xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2041 Switch derived parameters to use concurrent python for threading.

Browse source 
Change-Id: I791eb2f4803b5d29928dd153bbcf157ae643534f

Former-commit-id: 11f346d4d7baae08f61966a91e4d5f579630c7de
This commit is contained in:
Ben Steffensmeier 2013-06-04 16:41:00 -05:00
parent 778baf331f
commit 56f9b88ad5
10 changed files with 333 additions and 366 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF
View file

@ -17,6 +17,7 @@ Require-Bundle: com.raytheon.uf.common.status;bundle-version="1.12.1174",
org.jep;bundle-version="1.0.0",
org.eclipse.ui;bundle-version="3.6.1",
org.eclipse.core.runtime;bundle-version="3.6.0",
com.raytheon.uf.common.util;bundle-version="1.12.1174"
com.raytheon.uf.common.util;bundle-version="1.12.1174",
com.raytheon.uf.common.python.concurrent;bundle-version="1.0.0"
Export-Package: com.raytheon.uf.viz.derivparam.python,
com.raytheon.uf.viz.derivparam.python.function

112
cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java
View file

@ -22,20 +22,13 @@ package com.raytheon.uf.viz.derivparam.python;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import jep.JepException;
import java.util.concurrent.ExecutionException;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.LocalizationUtil;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.python.PyUtil;
import com.raytheon.uf.common.python.concurrent.PythonJobCoordinator;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -54,6 +47,8 @@ import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2010 mschenke Initial creation
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
*
* </pre>
*
@ -68,18 +63,10 @@ public class DerivParamPythonFunctionAdapter implements
private static final String PYTHON = "python";
private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR
+ File.separator
+ PYTHON
+ File.separator
+ "DerivParamImporter.py";
private static final String TEMPLATE_FILE = DerivedParameterGenerator.DERIV_PARAM_DIR
+ File.separator + PYTHON + File.separator + "functionTemplate.txt";
private MasterDerivScript masterScript;
private List<IDataRecord> results;
private PythonJobCoordinator<MasterDerivScript> coordinator;
/*
* (non-Javadoc)
@ -134,58 +121,11 @@ public class DerivParamPythonFunctionAdapter implements
*/
@Override
public void init() {
IPathManager pm = PathManagerFactory.getPathManager();
File script = pm.getStaticFile(INTERFACE_SCRIPT);
// Get list of all files for search hierarch of CAVE_STATIC
LocalizationFile[] derivParamFiles = pm.listFiles(
pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC),
DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false);
List<String> functionDirs = new ArrayList<String>(
derivParamFiles.length);
functionDirs.add(script.getParent());
Arrays.sort(derivParamFiles);
for (LocalizationFile file : derivParamFiles) {
if (file.isDirectory()
&& DerivedParameterGenerator.FUNCTIONS
.equals(LocalizationUtil.extractName(file.getName()))) {
// If it is a derived parameters functions directory, add to search list
functionDirs.add(file.getFile().getAbsolutePath());
}
}
// Create path from function dir list
String PATH = PyUtil.buildJepIncludePath(functionDirs
.toArray(new String[functionDirs.size()]));
List<String> preEvals = new ArrayList<String>(2);
preEvals.add("import DerivParamImporter");
StringBuilder cmd = new StringBuilder(200);
cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter(");
// Pass in directories to search based on function directories
int size = functionDirs.size() - 1;
for (int i = size; i > 0; --i) {
if (i < size) {
cmd.append(", ");
}
cmd.append("'").append(functionDirs.get(i)).append("'");
}
cmd.append("))");
preEvals.add(cmd.toString());
try {
if (masterScript != null) {
masterScript.dispose();
}
masterScript = new MasterDerivScript(PATH,
MasterDerivScript.class.getClassLoader(), preEvals);
} catch (JepException e) {
statusHandler.handle(Priority.PROBLEM,
"Failed to load derived parameters", e);
if (coordinator != null) {
coordinator.shutdown();
}
coordinator = PythonJobCoordinator
.newInstance(new MasterDerivScriptFactory());
}
/*
@ -195,41 +135,27 @@ public class DerivParamPythonFunctionAdapter implements
* com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#executeFunction
* (java.lang.String, java.util.List)
*/
@SuppressWarnings("unchecked")
@Override
public void executeFunction(String name, List<Object> arguments) {
public List<IDataRecord> executeFunction(String name, List<Object> arguments)
throws ExecutionException {
try {
results = (List<IDataRecord>) masterScript.executeFunction(name,
arguments);
} catch (JepException e) {
statusHandler.handle(Priority.PROBLEM,
"Error executing derived parameter request", e);
results = null;
return coordinator.submitSyncJob(new MasterDerivScriptExecutor(
name, arguments));
} catch (InterruptedException e) {
throw new ExecutionException(e);
}
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#getRequestResults
* ()
* @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#shutdown()
*/
@Override
public List<IDataRecord> getRequestResults() {
return results;
public void shutdown() {
if (coordinator != null) {
coordinator.shutdown();
}
/*
* (non-Javadoc)
*
* @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#
* getRequestIdentifierResults()
*/
@Override
public Object getRequestIdentifierResults() {
// TODO: Not supported yet
return null;
}
}

65
cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java Normal file
View file

@ -0,0 +1,65 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.viz.derivparam.python;
import java.util.List;
import jep.JepException;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.common.python.concurrent.IPythonExecutor;
/**
* Executor for calling executeFunction on a MasterDerivScript
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 04, 2013 2041 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class MasterDerivScriptExecutor implements
IPythonExecutor<MasterDerivScript, List<IDataRecord>> {
private final String name;
private final List<Object> arguments;
public MasterDerivScriptExecutor(String name, List<Object> arguments) {
this.name = name;
this.arguments = arguments;
}
@SuppressWarnings("unchecked")
@Override
public List<IDataRecord> execute(MasterDerivScript script)
throws JepException {
return (List<IDataRecord>) script.executeFunction(name, arguments);
}
}

120
cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java Normal file
View file

@ -0,0 +1,120 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.viz.derivparam.python;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jep.JepException;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.LocalizationUtil;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.python.PyUtil;
import com.raytheon.uf.common.python.concurrent.AbstractPythonScriptFactory;
import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator;
/**
* Factory for creating and initializing MasterDerivScript.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 04, 2013 2041 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class MasterDerivScriptFactory extends
AbstractPythonScriptFactory<MasterDerivScript> {
private static final int MAX_THREADS = 2;
public static final String NAME = "DerivedParameterPython";
private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR
+ File.separator
+ "python"
+ File.separator
+ "DerivParamImporter.py";
public MasterDerivScriptFactory() {
super(NAME, MAX_THREADS);
}
@Override
public MasterDerivScript createPythonScript() throws JepException {
IPathManager pm = PathManagerFactory.getPathManager();
File script = pm.getStaticFile(INTERFACE_SCRIPT);
// Get list of all files for search hierarch of CAVE_STATIC
LocalizationFile[] derivParamFiles = pm.listFiles(
pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC),
DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false);
List<String> functionDirs = new ArrayList<String>(
derivParamFiles.length);
functionDirs.add(script.getParent());
Arrays.sort(derivParamFiles);
for (LocalizationFile file : derivParamFiles) {
if (file.isDirectory()
&& DerivedParameterGenerator.FUNCTIONS
.equals(LocalizationUtil.extractName(file.getName()))) {
// If it is a derived parameters functions directory, add to
// search list
functionDirs.add(file.getFile().getAbsolutePath());
}
}
// Create path from function dir list
String PATH = PyUtil.buildJepIncludePath(functionDirs
.toArray(new String[functionDirs.size()]));
List<String> preEvals = new ArrayList<String>(2);
preEvals.add("import DerivParamImporter");
StringBuilder cmd = new StringBuilder(200);
cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter(");
// Pass in directories to search based on function directories
int size = functionDirs.size() - 1;
for (int i = size; i > 0; --i) {
if (i < size) {
cmd.append(", ");
}
cmd.append("'").append(functionDirs.get(i)).append("'");
}
cmd.append("))");
preEvals.add(cmd.toString());
return new MasterDerivScript(PATH,
MasterDerivScript.class.getClassLoader(), preEvals);
}
}

24
cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java
View file

@ -20,6 +20,7 @@
package com.raytheon.uf.viz.derivparam;
import java.util.List;
import java.util.concurrent.ExecutionException;
import com.raytheon.uf.common.datastorage.records.IDataRecord;
import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument;
@ -34,6 +35,8 @@ import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2010 mschenke Initial creation
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
*
* </pre>
*
@ -76,24 +79,11 @@ public interface IDerivParamFunctionAdapter {
* @param arguments
* arguments to pass into function
*/
public void executeFunction(String name, List<Object> arguments);
public List<IDataRecord> executeFunction(String name, List<Object> arguments)
throws ExecutionException;
/**
* Implementing Adapters need to be able to return the last executed
* function as data records copied into java from whatever mechanism was
* used to execute the function
*
* @return The last executed function results as data records
* Stop the adapter and any child threads.
*/
public List<IDataRecord> getRequestResults();
/**
* Implementing Adapters need to be able to return the last executed
* function as an object that the adapter can reuse in another function
* execution. This is to avoid copying data in and out of java constantly
*
* @return The last executed function results as an object the adapter can
* reuse
*/
public Object getRequestIdentifierResults();
public void shutdown();
}

13
cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java
View file

@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import com.raytheon.uf.common.dataplugin.level.Level;
import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
@ -43,6 +44,8 @@ import com.raytheon.uf.viz.derivparam.tree.CubeLevel;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 17, 2010 bsteffen Initial creation
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
*
* </pre>
*
@ -73,11 +76,12 @@ public class DerivedRequestableData extends AbstractRequestableData {
@Override
public Object getDataValue(Object arg) throws VizException {
DerivedParameterRequest request = createDerparRequest(arg);
DerivedParameterGenerator.addTask(request);
List<?> finalResult = request.getQueue();
try {
List<?> finalResult = DerivedParameterGenerator.calculate(request);
if (finalResult != null && finalResult.size() > 0) {
if (finalResult.size() == 1) {
return new IDataRecord[] { ((IDataRecord) finalResult.get(0)) };
return new IDataRecord[] { ((IDataRecord) finalResult
.get(0)) };
} else if (finalResult.size() == 4) {
return new IDataRecord[] {
((FloatDataRecord) finalResult.get(0)),
@ -89,6 +93,9 @@ public class DerivedRequestableData extends AbstractRequestableData {
"Error processing derived parameter, expecting scalar or vector data. Vector data must return speed, dir, u, and v components.");
}
}
} catch (ExecutionException e) {
throw new VizException("Error executing Derived Parameter.", e);
}
return null;
}

246
cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java
View file

@ -20,14 +20,15 @@
package com.raytheon.uf.viz.derivparam.library;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ExecutionException;
import javax.measure.unit.Unit;
import javax.xml.bind.JAXBException;
@ -77,10 +78,13 @@ import com.raytheon.uf.viz.derivparam.library.DerivParamMethod.MethodType;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 3, 2008 brockwoo Initial creation
* Nov 16, 2009 #3120 rjpeter Removed use of LevelNameMappingFile.
* Nov 20, 2009 #3387 jelkins Use derived script's variableId instead of filename
* Nov 21, 2009 #3576 rjpeter Refactored DerivParamDesc.
* Jul 03, 2008 brockwoo Initial creation
* Nov 16, 2009 3120 rjpeter Removed use of LevelNameMappingFile.
* Nov 20, 2009 3387 jelkins Use derived script's variableId instead
* of filename
* Nov 21, 2009 3576 rjpeter Refactored DerivParamDesc.
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
* </pre>
*
* @author brockwoo
@ -113,9 +117,41 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
// TODO: Handle multiple function types (python mixed with
// gsl/cuda/anything)
private static DerivParamFunctionType functionType;
private IDerivParamFunctionAdapter adapter;
static {
private Set<DerivParamUpdateListener> listeners = new HashSet<DerivParamUpdateListener>();
private Map<String, DerivParamDesc> derParLibrary;
private boolean needsLibInit = true;
private String extension = null;
private Job notifyJob = new Job("Notify Derived Parameter Listeners") {
@Override
protected IStatus run(IProgressMonitor arg0) {
Collection<DerivParamUpdateListener> l = null;
synchronized (listeners) {
l = new ArrayList<DerivParamUpdateListener>(listeners);
}
for (DerivParamUpdateListener listener : l) {
listener.updateDerParLibrary(derParLibrary);
}
return Status.OK_STATUS;
}
};
public static synchronized DerivedParameterGenerator getInstance() {
if (instance == null) {
instance = new DerivedParameterGenerator();
}
return instance;
}
public static DerivParamFunctionType[] getFunctionTypes() {
List<DerivParamFunctionType> functionTypes = new ArrayList<DerivParamFunctionType>();
IExtensionRegistry registry = Platform.getExtensionRegistry();
if (registry != null) {
IExtensionPoint point = registry.getExtensionPoint(EXTENSION);
@ -134,54 +170,15 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
functionType
.setAdapter((IDerivParamFunctionAdapter) cfg
.createExecutableExtension("adapter"));
DerivedParameterGenerator.functionType = functionType;
break;
functionTypes.add(functionType);
} catch (Throwable t) {
}
}
if (functionType != null) {
break;
}
}
if (functionType == null) {
statusHandler.handle(Priority.PROBLEM,
"Error creating derived parameter function type,"
+ " derived paramters will not be available");
statusHandler.handle(Priority.DEBUG,
t.getLocalizedMessage(), t);
}
}
}
private Set<DerivParamUpdateListener> listeners = new HashSet<DerivParamUpdateListener>();
private Map<String, DerivParamDesc> derParLibrary;
private BlockingQueue<DerivedParameterRequest> toDoList;
private static int numJobs = 2;
private DerivedParameterJob[] jobs = new DerivedParameterJob[numJobs];
private boolean needsLibInit = true;
private boolean runtime = true;
private String extension = null;
public static synchronized DerivedParameterGenerator getInstance() {
if (instance == null) {
instance = new DerivedParameterGenerator();
}
return instance;
}
public static DerivParamFunctionType[] getFunctionTypes() {
DerivedParameterGenerator gen = getInstance();
if (gen != null && gen.jobs.length > 0) {
return new DerivParamFunctionType[] { instance.jobs[0].functionType };
}
return new DerivParamFunctionType[0];
return functionTypes.toArray(new DerivParamFunctionType[0]);
}
public static synchronized Map<String, DerivParamDesc> getDerParLibrary() {
@ -189,52 +186,23 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
}
public static void registerUpdateListener(DerivParamUpdateListener listener) {
getInstance().listeners.add(listener);
DerivedParameterGenerator instance = getInstance();
synchronized (instance.listeners) {
instance.listeners.add(listener);
}
}
private DerivedParameterGenerator() {
for (int i = 0; i < jobs.length; i++) {
jobs[i] = new DerivedParameterJob("DerivParam Engine " + (i + 1));
}
DerivParamFunctionType[] functionTypes = getFunctionTypes();
// load the extensions
IExtensionRegistry registry = Platform.getExtensionRegistry();
if (registry != null) {
IExtensionPoint point = registry.getExtensionPoint(EXTENSION);
IExtension[] extensions = point.getExtensions();
for (IExtension ext : extensions) {
IConfigurationElement[] config = ext.getConfigurationElements();
for (DerivedParameterJob job : jobs) {
for (IConfigurationElement cfg : config) {
try {
DerivParamFunctionType functionType = new DerivParamFunctionType();
functionType.setName(cfg.getAttribute("name"));
functionType.setExtension(cfg
.getAttribute("extension"));
this.extension = functionType.getExtension();
functionType
.setAdapter((IDerivParamFunctionAdapter) cfg
.createExecutableExtension("adapter"));
job.functionType = functionType;
break;
} catch (Throwable t) {
}
}
if (job.functionType == null) {
statusHandler
.handle(Priority.PROBLEM,
if (functionTypes == null || functionTypes.length == 0) {
statusHandler.handle(Priority.PROBLEM,
"Error creating derived parameter function type,"
+ " derived paramters will not be available");
break;
}
}
}
}
toDoList = new LinkedBlockingQueue<DerivedParameterRequest>();
this.adapter = functionTypes[0].getAdapter();
this.extension = functionTypes[0].getExtension();
notifyJob.setSystem(true);
LocalizationFile dir = PathManagerFactory.getPathManager()
.getStaticLocalizationFile(DERIV_PARAM_DIR);
if (dir != null) {
@ -242,9 +210,6 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
}
initLibrary();
for (int i = 0; i < jobs.length; i++) {
jobs[i].schedule();
}
}
/**
@ -254,11 +219,13 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
* A derived parameter request
* @return boolean indicating if the request was put into queue
*/
public static boolean addTask(DerivedParameterRequest task) {
return getInstance().toDoList.add(task);
public static List<IDataRecord> calculate(DerivedParameterRequest task)
throws ExecutionException {
return getInstance().adapter.executeFunction(task.getMethod(),
Arrays.asList(task.getArgumentRecords()));
}
private void initLibrary() {
private synchronized void initLibrary() {
if (needsLibInit) {
long start = System.currentTimeMillis();
Set<String> derivParamFiles = new HashSet<String>();
@ -334,11 +301,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
}
}
this.derParLibrary = derParLibrary;
for (DerivParamUpdateListener listener : listeners) {
listener.updateDerParLibrary(derParLibrary);
}
adapter.init();
System.out.println("time to init derived parameter thread: "
notifyJob.schedule();
System.out.println("time to init derived parameters: "
+ (System.currentTimeMillis() - start) + "ms");
needsLibInit = false;
}
@ -359,86 +326,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
@Override
public void fileUpdated(FileUpdatedMessage message) {
needsLibInit = true;
for (DerivedParameterJob job : jobs) {
job.needsInit = true;
}
initLibrary();
}
public static void shutdown() {
if (instance != null) {
instance.runtime = false;
}
getInstance().adapter.shutdown();
}
private class DerivedParameterJob extends Job {
// TODO: Handle multiple function types (python mixed with
// gsl/cuda/anything)
private DerivParamFunctionType functionType;
private boolean needsInit = true;
/**
* @param name
*/
public DerivedParameterJob(String name) {
super(name);
this.setSystem(true);
}
private void init() {
if (needsInit) {
functionType.getAdapter().init();
DerivedParameterGenerator.this.initLibrary();
needsInit = false;
}
}
/*
* (non-Javadoc)
*
* @seeorg.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime.
* IProgressMonitor)
*/
@Override
protected IStatus run(IProgressMonitor monitor) {
while (runtime) {
if (derParLibrary == null) {
this.init();
}
DerivedParameterRequest request = null;
try {
request = toDoList.take();
} catch (InterruptedException e) {
}
if (needsInit) {
init();
}
if (request != null) {
// long t0 = System.currentTimeMillis();
try {
functionType.getAdapter().executeFunction(
request.getMethod(),
Arrays.asList(request.getArgumentRecords()));
List<IDataRecord> result = functionType.getAdapter()
.getRequestResults();
for (IDataRecord rec : result) {
rec.setName(request.getParameterAbbreviation());
}
request.setQueue(result);
} catch (Throwable e) {
statusHandler
.handle(Priority.PROBLEM,
"Derived Parameter Engine failed to generate parameter",
e);
}
// System.out.println(getName() + " calc took: "
// + (System.currentTimeMillis() - t0));
}
}
return Status.OK_STATUS;
}
}
}

47
cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java
View file

@ -21,11 +21,8 @@ package com.raytheon.uf.viz.derivparam.library;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
import com.raytheon.uf.common.time.DataTime;
import com.raytheon.uf.viz.core.exception.VizException;
/**
* The DerivedParameteRequest is the way for a class to communicate a request
@ -37,7 +34,9 @@ import com.raytheon.uf.viz.core.exception.VizException;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 9, 2008 brockwoo Initial creation
* Jul 09, 2008 brockwoo Initial creation
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
*
* </pre>
*
@ -56,10 +55,7 @@ public class DerivedParameterRequest {
private DataTime baseTime;
private SynchronousQueue<List<?>> queue;
public DerivedParameterRequest() {
this.queue = new SynchronousQueue<List<?>>();
this.baseParam = new ArrayList<Object>();
}
@ -164,41 +160,4 @@ public class DerivedParameterRequest {
this.argumentRecords = argumentRecords;
}
/**
* This method will return the data calculated by the derived parameter
* script. It is important to call this method right after the request has
* been submitted to the derived parameter thread as it will block for 30
* seconds waiting for the response.
*
* @return the raw data calculated by the script
*/
public List<?> getQueue() throws VizException {
try {
List<?> result = queue.poll(30, TimeUnit.SECONDS);
if (result == null) {
throw new VizException("Derived Parameter Timed Out");
}
return result;
} catch (InterruptedException e) {
throw new VizException(e);
}
}
/**
* Used by the derived parameter generator to return the calculated data to
* the thread requesting it. This will block until <code>getQueue</code> is
* called.
*
* @param queue
* the data being returned to the thread making the request
* @throws VizException
*/
public void setQueue(List<?> queue) {
try {
this.queue.offer(queue, 15, TimeUnit.SECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

12
cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java
View file

@ -22,6 +22,7 @@ package com.raytheon.uf.viz.objectiveanalysis.rsc;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import org.geotools.coverage.grid.GeneralGridGeometry;
import org.opengis.referencing.crs.CoordinateReferenceSystem;
@ -54,6 +55,8 @@ import com.raytheon.viz.grid.util.TiltUtils;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 20, 2010 bsteffen Initial creation
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
*
* </pre>
*
@ -210,9 +213,12 @@ public class OATiltGridTransformer extends OAGridTransformer {
DerivedParameterRequest sliceRequest = new DerivedParameterRequest();
sliceRequest.setMethod("Slice");
sliceRequest.setArgumentRecords(new Object[] { cube, presRequest, -1 });
DerivedParameterGenerator.addTask(sliceRequest);
return ((FloatDataRecord) sliceRequest.getQueue().get(0))
.getFloatData();
try {
return ((FloatDataRecord) DerivedParameterGenerator.calculate(
sliceRequest).get(0)).getFloatData();
} catch (ExecutionException e) {
throw new VizException(e);
}
}

17
cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java
View file

@ -35,6 +35,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import javax.media.jai.Interpolation;
@ -84,11 +85,12 @@ import com.raytheon.uf.viz.derivparam.library.IDerivParamField;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 27, 2009 jsanchez Initial creation
* Nov 21, 2009 #3576 rjpeter Refactored use of DerivParamDesc.
* - AWIPS2 Baseline Repository --------
* 08/03/2012 798 jkorman Explicitly set interpolationLevels
* from "source" record.
* 04/08/2013 #1293 bkowal Removed references to hdffileid.
* Nov 21, 2009 3576 rjpeter Refactored use of DerivParamDesc.
* Aug 03, 2012 798 jkorman Explicitly set interpolationLevels from
* "source" record.
* Apr 08, 2013 1293 bkowal Removed references to hdffileid.
* Jun 04, 2013 2041 bsteffen Switch derived parameters to use
* concurrent python for threading.
* </pre>
*
* @author jsanchez
@ -313,11 +315,10 @@ public class SatelliteDataCubeAdapter implements IDataCubeAdapter {
}
derivedRequest.setArgumentRecords(bytes.toArray(new Object[] {}));
DerivedParameterGenerator.addTask(derivedRequest);
List<?> finalResult;
try {
finalResult = derivedRequest.getQueue();
} catch (VizException e) {
finalResult = DerivedParameterGenerator.calculate(derivedRequest);
} catch (ExecutionException e) {
throw new VizDataCubeException(e);
}