xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #766: Merge VCModule refactor and Python globals memory leak fix to common baseline.

Browse source 
Former-commit-id: 672d7a2c31 [formerly 48c3d7d8c7cec6c7366705c9318ce35a46bd8b3d]
Former-commit-id: 2a0ffc5311
This commit is contained in:
David Gillingham 2012-06-25 15:31:38 -05:00
parent bfc4e1da3c
commit 768afb8423
9 changed files with 271 additions and 161 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/IParmManager.java
View file

@ -38,6 +38,7 @@ import com.raytheon.viz.gfe.core.msgs.IParmInventoryChangedListener;
import com.raytheon.viz.gfe.core.msgs.IParmListChangedListener;
import com.raytheon.viz.gfe.core.msgs.ISystemTimeRangeChangedListener;
import com.raytheon.viz.gfe.core.parm.Parm;
import com.raytheon.viz.gfe.core.parm.vcparm.VCModuleJobPool;
/**
* Placeholder for ParmManager interface
@ -47,6 +48,7 @@ import com.raytheon.viz.gfe.core.parm.Parm;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/28/2008 chammack Initial creation of skeleton.
* 06/25/2012 #766 dgilling Added getVCModulePool().
*
* </pre>
*
@ -570,4 +572,6 @@ public interface IParmManager extends IParmInventoryChangedListener,
public ParmID fromExpression(String parmName);
public JobPool getNotificationPool();
public VCModuleJobPool getVCModulePool();
}

13
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/internal/AbstractParmManager.java
View file

@ -74,6 +74,7 @@ import com.raytheon.viz.gfe.core.msgs.ShowISCGridsMsg;
import com.raytheon.viz.gfe.core.parm.ABVParmID;
import com.raytheon.viz.gfe.core.parm.Parm;
import com.raytheon.viz.gfe.core.parm.vcparm.VCModule;
import com.raytheon.viz.gfe.core.parm.vcparm.VCModuleJobPool;
/**
* Implements common parm manager functionality shared between concrete and mock
@ -92,6 +93,8 @@ import com.raytheon.viz.gfe.core.parm.vcparm.VCModule;
* 03/01/2012 #354 dgilling Modify setParms to always load (but not
* necessarily display) the ISC parms that
* correspond to a visible mutable parm.
* 06/25/2012 #766 dgilling Move to a shared thread pool for VCModule
* execution.
*
* </pre>
*
@ -244,6 +247,8 @@ public abstract class AbstractParmManager implements IParmManager {
private JobPool notificationPool;
private VCModuleJobPool vcModulePool;
protected AbstractParmManager(final DataManager dataManager) {
this.dataManager = dataManager;
this.parms = new RWLArrayList<Parm>();
@ -258,6 +263,8 @@ public abstract class AbstractParmManager implements IParmManager {
// Get virtual parm definitions
vcModules = initVirtualCalcParmDefinitions();
vcModulePool = new VCModuleJobPool("GFE Virtual ISC Python executor",
this.dataManager, vcModules.size(), Boolean.TRUE);
PythonPreferenceStore prefs = Activator.getDefault()
.getPreferenceStore();
@ -413,6 +420,7 @@ public abstract class AbstractParmManager implements IParmManager {
parms.releaseReadLock();
}
vcModulePool.cancel();
for (VCModule module : vcModules) {
module.dispose();
}
@ -2060,4 +2068,9 @@ public abstract class AbstractParmManager implements IParmManager {
public JobPool getNotificationPool() {
return notificationPool;
}
@Override
public VCModuleJobPool getVCModulePool() {
return vcModulePool;
}
}

8
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/internal/ParmManager.java
View file

@ -71,6 +71,8 @@ import com.raytheon.viz.gfe.types.MutableInteger;
* 08/19/09 2547 rjpeter Implement Test/Prac database display.
* 02/23/12 #346 dgilling Call Parm's dispose method when removing
* a Parm.
* 06/25/12 #766 dgilling Fix NullPointerException from VCModules
* when running in practice mode.
* </pre>
*
* @author bphillip
@ -437,6 +439,12 @@ public class ParmManager extends AbstractParmManager {
@Override
public ParmID[] getAvailableParms(DatabaseID dbID) {
// a derivation from AWIPS1:
// short-circuit the checks and just return an empty array back
// if we have an invalid DatabaseID
if ((dbID == null) || (!dbID.isValid())) {
return new ParmID[0];
}
// Check the cache
List<ParmID> cacheParmIDs = null;

9
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/parm/VCParm.java
View file

@ -67,6 +67,9 @@ import com.raytheon.viz.gfe.core.parm.vcparm.VCModule.VCInventory;
* Mar 02, 2012 #346 dgilling Use Parm's new disposed flag to
* prevent leaks through
* ListenerLists.
* Jun 25, 2012 #766 dgilling Cleanup error logging so we
* don't spam alertViz in practice
* mode.
*
* </pre>
*
@ -104,7 +107,7 @@ public class VCParm extends VParm implements IParmListChangedListener,
// Need to check that the above call to mod.getGpi() did not fail
if (!mod.isValid()) {
statusHandler.handle(Priority.PROBLEM, "Can't get GPI: ",
statusHandler.handle(Priority.EVENTB, "Can't get GPI: ",
this.mod.getErrorString());
}
@ -410,7 +413,7 @@ public class VCParm extends VParm implements IParmListChangedListener,
// ensure we have parms* for all of the dependent parms
List<ParmID> args = new ArrayList<ParmID>(mod.dependentParms());
if (!mod.isValid()) {
statusHandler.handle(Priority.PROBLEM,
statusHandler.handle(Priority.EVENTB,
"Error getting dependent WeatherElements: ",
mod.getErrorString());
}
@ -455,7 +458,7 @@ public class VCParm extends VParm implements IParmListChangedListener,
// get list of dependent parms
List<ParmID> args = new ArrayList<ParmID>(mod.dependentParms());
if (!mod.isValid()) {
statusHandler.handle(Priority.PROBLEM,
statusHandler.handle(Priority.EVENTB,
"Error getting dependent WeatherElements: ",
mod.getErrorString());
}

41
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/parm/vcparm/VCModule.java
View file

@ -70,6 +70,8 @@ import com.raytheon.viz.gfe.core.parm.Parm;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 17, 2011 dgilling Initial creation
* Jun 20, 2012 #766 dgilling Refactor to improve
* performance.
*
* </pre>
*
@ -149,7 +151,7 @@ public class VCModule {
private GridParmInfo gpi;
private VCModuleJob python;
private Collection<ParmID> depParms;
private DataManager dataMgr;
@ -161,12 +163,11 @@ public class VCModule {
this.dataMgr = dataMgr;
this.parmMgr = parmMgr;
this.id = module.getName().split("\\.(?=[^\\.]+$)")[0];
this.python = new VCModuleJob(this.dataMgr);
this.python.schedule();
this.depParms = Collections.emptyList();
}
public void dispose() {
this.python.cancel();
// no-op
}
public boolean isValid() {
@ -185,7 +186,7 @@ public class VCModule {
Map<String, Object> args = new HashMap<String, Object>();
args.put(PyConstants.METHOD_NAME, method);
VCModuleRequest req = new VCModuleRequest(id, "getMethodArgs", args);
python.enqueue(req);
parmMgr.getVCModulePool().enqueue(req);
Object result = req.getResult();
String[] argNames = (String[]) result;
@ -193,14 +194,20 @@ public class VCModule {
}
public Collection<ParmID> dependentParms() {
Collection<ParmID> rval = new ArrayList<ParmID>();
// this is a derivation from AWIPS1
// like getGpi(), this should only ever need to be calculated once
// since VCModule does not support dynamic updates.
if (!depParms.isEmpty()) {
return depParms;
}
try {
Collection<String> parameters = getMethodArgs("getInventory");
depParms = new ArrayList<ParmID>(parameters.size());
for (String parmName : parameters) {
ParmID pid = parmMgr.fromExpression(parmName);
if (pid.isValid()) {
rval.add(pid);
depParms.add(pid);
} else {
throw new IllegalArgumentException(
"Can't find Weather Element for " + parmName);
@ -210,10 +217,10 @@ public class VCModule {
error = t;
// statusHandler.handle(Priority.DEBUG, "dependentParms: " + id
// + " error", t);
return Collections.emptyList();
depParms = Collections.emptyList();
}
return rval;
return depParms;
}
private long[] encodeTR(final TimeRange tr) {
@ -235,9 +242,9 @@ public class VCModule {
Object[] item = new Object[3];
item[0] = encodeTR(gd.getGridTime());
// since we have to go through a bunch of hoops in VCModuleScript to get
// the IGridData in python-useable format, no need doing anything here
// but storing the data
// since we have to go through a bunch of hoops in VCModuleController to
// get the IGridData in python-useable format, no need doing anything
// here but storing the data
item[1] = gd;
// add a mask indicating the set of valid points. Note for all data
@ -328,7 +335,7 @@ public class VCModule {
}
VCModuleRequest req = new VCModuleRequest(id, "getInventory", cargs);
python.enqueue(req);
parmMgr.getVCModulePool().enqueue(req);
Object reqResult = req.getResult();
// what's returned from the script here is a list of tuples.
@ -389,7 +396,7 @@ public class VCModule {
// commenting out this python call because it is completely
// superfluous--all the baseline VCMODULE files have a calcHistory
// method so there's no point in checking and it saves a call into the
// VCModuleJob queue. If at some point there's a desire to support
// VCModuleJobPool queue. If at some point there's a desire to support
// user/site-defined modules, this check should probably return.
// TODO: Reimplement using a call to BaseGfePyController.hasMethod().
@ -442,7 +449,7 @@ public class VCModule {
}
VCModuleRequest req = new VCModuleRequest(id, "calcHistory", cargs);
python.enqueue(req);
parmMgr.getVCModulePool().enqueue(req);
Object reqResult = req.getResult();
List<String> result = (List<String>) reqResult;
@ -500,7 +507,7 @@ public class VCModule {
VCModuleRequest req = new VCModuleRequest(id, "calcGrid", cargs,
getGpi().getGridType());
python.enqueue(req);
parmMgr.getVCModulePool().enqueue(req);
Object reqResult = req.getResult();
return decodeGD(reqResult, invEntry);
@ -521,7 +528,7 @@ public class VCModule {
try {
VCModuleRequest req = new VCModuleRequest(id, "getWEInfo", null);
python.enqueue(req);
parmMgr.getVCModulePool().enqueue(req);
Object reqResult = req.getResult();
List<List<Object>> result = (List<List<Object>>) reqResult;

134
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/parm/vcparm/VCModuleJob.java
View file

@ -1,134 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.viz.gfe.core.parm.vcparm;
import java.util.concurrent.TimeUnit;
import jep.JepException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import com.raytheon.uf.common.dataplugin.gfe.StatusConstants;
import com.raytheon.uf.common.python.PyConstants;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.viz.core.jobs.AbstractQueueJob;
import com.raytheon.viz.gfe.core.DataManager;
/**
* <code>Job</code> which allows <code>VCModule</code> python calls to run off a
* common thread.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 20, 2011 dgilling Initial creation
*
* </pre>
*
* @author dgilling
* @version 1.0
*/
public class VCModuleJob extends AbstractQueueJob<VCModuleRequest> {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(VCModuleJob.class);
private VCModuleController python = null;
private DataManager dataMgr;
public VCModuleJob(DataManager dataMgr) {
super("GFE Virtual ISC Python executor");
setSystem(true);
this.dataMgr = dataMgr;
}
/*
* (non-Javadoc)
*
* @see org.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime.
* IProgressMonitor)
*/
@Override
protected IStatus run(IProgressMonitor monitor) {
try {
try {
python = VCModuleControllerFactory.buildInstance(dataMgr);
} catch (JepException e) {
return new Status(IStatus.ERROR, StatusConstants.PLUGIN_ID,
"Error initializing VCModule python object", e);
}
while (!monitor.isCanceled()) {
VCModuleRequest request = null;
try {
request = queue.poll(1000L, TimeUnit.MILLISECONDS);
if (request != null) {
processRequest(request);
}
// TODO: Reinstate this call, if we ever want to support
// dynamic editing of VCMODULE files through the
// Localization perspective.
// python.processFileUpdates();
} catch (InterruptedException e) {
statusHandler.handle(Priority.PROBLEM,
"VC Module python thread interrupted.", e);
break;
} catch (Throwable t) {
// statusHandler.handle(Priority.DEBUG,
// "Error running VCModule method.", t);
request.setResult(t);
}
}
} finally {
if (python != null) {
python.dispose();
python = null;
}
}
return Status.OK_STATUS;
}
private void processRequest(VCModuleRequest request) throws JepException {
Object result = null;
if (request.getMethodName().equals("getMethodArgs")) {
result = python.getMethodArguments(request.getModuleName(),
(String) request.getArgMap().get(PyConstants.METHOD_NAME));
} else {
result = python.executeMethod(request.getModuleName(),
request.getMethodName(), request.getJepArgs(),
request.getType());
}
request.setResult(result);
}
}

207
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/parm/vcparm/VCModuleJobPool.java Normal file
View file

@ -0,0 +1,207 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.viz.gfe.core.parm.vcparm;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import jep.JepException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import com.raytheon.uf.common.python.PyConstants;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.viz.gfe.core.DataManager;
/**
* TODO Add Description
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 22, 2012 dgilling Initial creation
*
* </pre>
*
* @author dgilling
* @version 1.0
*/
public class VCModuleJobPool {
protected static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(VCModuleJobPool.class);
protected LinkedBlockingQueue<VCModuleRequest> workQueue = new LinkedBlockingQueue<VCModuleRequest>();
protected List<Job> jobList;
public VCModuleJobPool(String name, DataManager dataMgr, int size) {
this(name, dataMgr, size, null, null);
}
public VCModuleJobPool(String name, DataManager dataMgr, int size,
Boolean system) {
this(name, dataMgr, size, system, null);
}
public VCModuleJobPool(String name, DataManager dataMgr, int size,
Boolean system, Integer priority) {
jobList = new ArrayList<Job>(size);
for (int i = 0; i < size; i++) {
PooledJob job = new PooledJob(name, null);
if (system != null) {
job.setSystem(system);
}
if (priority != null) {
job.setPriority(priority);
}
jobList.add(job);
job.schedule();
}
}
public synchronized void enqueue(VCModuleRequest request) {
workQueue.offer(request);
}
/**
* Join on the <code>Job</code>s in the pool. Attempting to schedule other
* <code>Job</code>s will block until join as returned so be careful when
* calling
*/
public synchronized void join() {
for (Job j : jobList) {
try {
j.join();
} catch (InterruptedException e) {
// Ignore interrupt
}
}
}
/**
* Cancel the job pool, will clear out the workQueue then join on all jobs
* running
*/
public synchronized void cancel() {
workQueue.clear();
for (Job job : jobList) {
job.cancel();
}
join();
}
/**
* Cancels the specified request. Returns true if the provided runnable was
* waiting to be run but now is now. Returns false if the provided runnable
* is already running or if it was not enqueued to begin with.
*
* @param request
* @return
*/
public synchronized boolean cancel(VCModuleRequest request) {
return workQueue.remove(request);
}
protected class PooledJob extends Job {
protected DataManager dataMgr;
protected VCModuleController python;
public PooledJob(String name, DataManager dataMgr) {
super(name);
this.dataMgr = dataMgr;
this.python = null;
}
@Override
protected IStatus run(IProgressMonitor monitor) {
try {
if (python == null) {
python = VCModuleControllerFactory.buildInstance(dataMgr);
}
while (!monitor.isCanceled()) {
try {
VCModuleRequest request = workQueue.poll(1L,
TimeUnit.SECONDS);
if (request != null) {
processRequest(request);
}
} catch (InterruptedException e) {
// ignore, but log
statusHandler.handle(
Priority.DEBUG,
"VCModuleJobPool received interrupt: "
+ e.getLocalizedMessage(), e);
}
}
} catch (JepException e) {
statusHandler.handle(
Priority.WARN,
"Could not instantiate VCMoudleController: "
+ e.getLocalizedMessage(), e);
} finally {
if (python != null) {
python.dispose();
}
}
return Status.CANCEL_STATUS;
}
protected void processRequest(VCModuleRequest request) {
Object result = null;
try {
if (request.getMethodName().equals("getMethodArgs")) {
result = python.getMethodArguments(
request.getModuleName(),
(String) request.getArgMap().get(
PyConstants.METHOD_NAME));
} else {
result = python.executeMethod(request.getModuleName(),
request.getMethodName(), request.getJepArgs(),
request.getType());
}
} catch (Throwable t) {
statusHandler.handle(
Priority.DEBUG,
"Exception thrown in VCModule's python.execute(): "
+ t.getLocalizedMessage(), t);
result = t;
}
request.setResult(result);
}
}
}

7
edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/db/objects/DatabaseID.java
View file

@ -47,8 +47,9 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeTypeAdap
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 3/6/08 875 bphillip Initial Creation
* 8/19/09 2899 njensen Rewrote equals() for performance
* 5/08/12 #600 dgilling Implement clone().
* 8/19/09 2899 njensen Rewrote equals() for performance
* 5/08/12 #600 dgilling Implement clone().
* 6/25/12 #766 dgilling Fix isValid().
*
* </pre>
*
@ -228,7 +229,7 @@ public class DatabaseID implements Serializable, Comparable<DatabaseID>,
*/
public boolean isValid() {
return !this.format.equals("NONE");
return !this.format.equals(DataType.NONE);
}
/**

9
edexOsgi/com.raytheon.uf.common.python/src/com/raytheon/uf/common/python/PythonScript.java
View file

@ -54,8 +54,9 @@ import jep.JepException;
* Feb 4, 2008 njensen Initial creation
* Mar 21, 2008 njensen Major refactor
* June 9, 2008 njensen Refactor
* Sep 18, 2009 2899 njensen Added cProfile support
* Dec 7, 2009 3310 njensen Separated some functionality up to PythonInterpreter
* Sep 18, 2009 2899 njensen Added cProfile support
* Dec 7, 2009 3310 njensen Separated some functionality up to PythonInterpreter
* Jun 26, 2012 #776 dgilling Fix leaking of global names.
*
* </pre>
*
@ -244,10 +245,10 @@ public class PythonScript extends PythonInterpreter {
}
protected void cleanupArgs(List<String> args) throws JepException {
if (args != null && args.size() > 0) {
if (args != null && !args.isEmpty()) {
for (String key : args) {
if (!key.equals("self")) {
jep.eval(key + " = None");
jep.eval("del " + key);
}
}
}