From 56f9b88ad537c6a18fe9c4592ba286710fe0fe4d Mon Sep 17 00:00:00 2001 From: Ben Steffensmeier Date: Tue, 4 Jun 2013 16:41:00 -0500 Subject: [PATCH] Issue #2041 Switch derived parameters to use concurrent python for threading. Change-Id: I791eb2f4803b5d29928dd153bbcf157ae643534f Former-commit-id: 11f346d4d7baae08f61966a91e4d5f579630c7de --- .../META-INF/MANIFEST.MF | 3 +- .../DerivParamPythonFunctionAdapter.java | 116 ++------ .../python/MasterDerivScriptExecutor.java | 65 +++++ .../python/MasterDerivScriptFactory.java | 120 +++++++++ .../IDerivParamFunctionAdapter.java | 26 +- .../data/DerivedRequestableData.java | 37 +-- .../library/DerivedParameterGenerator.java | 252 +++++------------- .../library/DerivedParameterRequest.java | 49 +--- .../rsc/OATiltGridTransformer.java | 14 +- .../satellite/SatelliteDataCubeAdapter.java | 17 +- 10 files changed, 333 insertions(+), 366 deletions(-) create mode 100644 cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java create mode 100644 cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java diff --git a/cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF b/cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF index 1a0f1e7664..36a778a9ea 100644 --- a/cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF +++ b/cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF @@ -17,6 +17,7 @@ Require-Bundle: com.raytheon.uf.common.status;bundle-version="1.12.1174", org.jep;bundle-version="1.0.0", org.eclipse.ui;bundle-version="3.6.1", org.eclipse.core.runtime;bundle-version="3.6.0", - com.raytheon.uf.common.util;bundle-version="1.12.1174" + com.raytheon.uf.common.util;bundle-version="1.12.1174", + com.raytheon.uf.common.python.concurrent;bundle-version="1.0.0" Export-Package: com.raytheon.uf.viz.derivparam.python, com.raytheon.uf.viz.derivparam.python.function diff --git a/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java index e3c2e486fd..70b1ca3caf 100644 --- a/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java +++ b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java @@ -22,20 +22,13 @@ package com.raytheon.uf.viz.derivparam.python; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.List; - -import jep.JepException; +import java.util.concurrent.ExecutionException; import com.raytheon.uf.common.datastorage.records.IDataRecord; -import com.raytheon.uf.common.localization.IPathManager; -import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; -import com.raytheon.uf.common.localization.LocalizationFile; -import com.raytheon.uf.common.localization.LocalizationUtil; import com.raytheon.uf.common.localization.PathManagerFactory; -import com.raytheon.uf.common.python.PyUtil; +import com.raytheon.uf.common.python.concurrent.PythonJobCoordinator; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; @@ -53,7 +46,9 @@ import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Dec 16, 2010 mschenke Initial creation + * Dec 16, 2010 mschenke Initial creation + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @@ -68,18 +63,10 @@ public class DerivParamPythonFunctionAdapter implements private static final String PYTHON = "python"; - private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR - + File.separator - + PYTHON - + File.separator - + "DerivParamImporter.py"; - private static final String TEMPLATE_FILE = DerivedParameterGenerator.DERIV_PARAM_DIR + File.separator + PYTHON + File.separator + "functionTemplate.txt"; - private MasterDerivScript masterScript; - - private List results; + private PythonJobCoordinator coordinator; /* * (non-Javadoc) @@ -134,58 +121,11 @@ public class DerivParamPythonFunctionAdapter implements */ @Override public void init() { - IPathManager pm = PathManagerFactory.getPathManager(); - - File script = pm.getStaticFile(INTERFACE_SCRIPT); - - // Get list of all files for search hierarch of CAVE_STATIC - LocalizationFile[] derivParamFiles = pm.listFiles( - pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC), - DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false); - List functionDirs = new ArrayList( - derivParamFiles.length); - functionDirs.add(script.getParent()); - - Arrays.sort(derivParamFiles); - - for (LocalizationFile file : derivParamFiles) { - if (file.isDirectory() - && DerivedParameterGenerator.FUNCTIONS - .equals(LocalizationUtil.extractName(file.getName()))) { - // If it is a derived parameters functions directory, add to search list - functionDirs.add(file.getFile().getAbsolutePath()); - } - } - - // Create path from function dir list - String PATH = PyUtil.buildJepIncludePath(functionDirs - .toArray(new String[functionDirs.size()])); - - List preEvals = new ArrayList(2); - preEvals.add("import DerivParamImporter"); - StringBuilder cmd = new StringBuilder(200); - cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter("); - // Pass in directories to search based on function directories - int size = functionDirs.size() - 1; - for (int i = size; i > 0; --i) { - if (i < size) { - cmd.append(", "); - } - cmd.append("'").append(functionDirs.get(i)).append("'"); - } - cmd.append("))"); - preEvals.add(cmd.toString()); - - try { - if (masterScript != null) { - masterScript.dispose(); - } - masterScript = new MasterDerivScript(PATH, - MasterDerivScript.class.getClassLoader(), preEvals); - } catch (JepException e) { - statusHandler.handle(Priority.PROBLEM, - "Failed to load derived parameters", e); + if (coordinator != null) { + coordinator.shutdown(); } + coordinator = PythonJobCoordinator + .newInstance(new MasterDerivScriptFactory()); } /* @@ -195,41 +135,27 @@ public class DerivParamPythonFunctionAdapter implements * com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#executeFunction * (java.lang.String, java.util.List) */ - @SuppressWarnings("unchecked") @Override - public void executeFunction(String name, List arguments) { + public List executeFunction(String name, List arguments) + throws ExecutionException { try { - results = (List) masterScript.executeFunction(name, - arguments); - } catch (JepException e) { - statusHandler.handle(Priority.PROBLEM, - "Error executing derived parameter request", e); - results = null; + return coordinator.submitSyncJob(new MasterDerivScriptExecutor( + name, arguments)); + } catch (InterruptedException e) { + throw new ExecutionException(e); } } /* * (non-Javadoc) * - * @see - * com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#getRequestResults - * () + * @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#shutdown() */ @Override - public List getRequestResults() { - return results; - } - - /* - * (non-Javadoc) - * - * @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter# - * getRequestIdentifierResults() - */ - @Override - public Object getRequestIdentifierResults() { - // TODO: Not supported yet - return null; + public void shutdown() { + if (coordinator != null) { + coordinator.shutdown(); + } } } diff --git a/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java new file mode 100644 index 0000000000..06e4b174d4 --- /dev/null +++ b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java @@ -0,0 +1,65 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.viz.derivparam.python; + +import java.util.List; + +import jep.JepException; + +import com.raytheon.uf.common.datastorage.records.IDataRecord; +import com.raytheon.uf.common.python.concurrent.IPythonExecutor; + +/** + * Executor for calling executeFunction on a MasterDerivScript + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 04, 2013 2041       bsteffen    Initial creation
+ * 
+ *
+ * + * @author bsteffen + * @version 1.0 + */ + +public class MasterDerivScriptExecutor implements + IPythonExecutor> { + + private final String name; + + private final List arguments; + + public MasterDerivScriptExecutor(String name, List arguments) { + this.name = name; + this.arguments = arguments; + } + + @SuppressWarnings("unchecked") + @Override + public List execute(MasterDerivScript script) + throws JepException { + return (List) script.executeFunction(name, arguments); + } + +} diff --git a/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java new file mode 100644 index 0000000000..e03fbcf7a9 --- /dev/null +++ b/cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java @@ -0,0 +1,120 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.viz.derivparam.python; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import jep.JepException; + +import com.raytheon.uf.common.localization.IPathManager; +import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType; +import com.raytheon.uf.common.localization.LocalizationFile; +import com.raytheon.uf.common.localization.LocalizationUtil; +import com.raytheon.uf.common.localization.PathManagerFactory; +import com.raytheon.uf.common.python.PyUtil; +import com.raytheon.uf.common.python.concurrent.AbstractPythonScriptFactory; +import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator; + +/** + * Factory for creating and initializing MasterDerivScript. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 04, 2013 2041       bsteffen    Initial creation
+ * 
+ *
+ * + * @author bsteffen + * @version 1.0 + */ + +public class MasterDerivScriptFactory extends + AbstractPythonScriptFactory { + + private static final int MAX_THREADS = 2; + + public static final String NAME = "DerivedParameterPython"; + + private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR + + File.separator + + "python" + + File.separator + + "DerivParamImporter.py"; + + public MasterDerivScriptFactory() { + super(NAME, MAX_THREADS); + } + + @Override + public MasterDerivScript createPythonScript() throws JepException { + IPathManager pm = PathManagerFactory.getPathManager(); + + File script = pm.getStaticFile(INTERFACE_SCRIPT); + + // Get list of all files for search hierarch of CAVE_STATIC + LocalizationFile[] derivParamFiles = pm.listFiles( + pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC), + DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false); + List functionDirs = new ArrayList( + derivParamFiles.length); + functionDirs.add(script.getParent()); + + Arrays.sort(derivParamFiles); + + for (LocalizationFile file : derivParamFiles) { + if (file.isDirectory() + && DerivedParameterGenerator.FUNCTIONS + .equals(LocalizationUtil.extractName(file.getName()))) { + // If it is a derived parameters functions directory, add to + // search list + functionDirs.add(file.getFile().getAbsolutePath()); + } + } + + // Create path from function dir list + String PATH = PyUtil.buildJepIncludePath(functionDirs + .toArray(new String[functionDirs.size()])); + + List preEvals = new ArrayList(2); + preEvals.add("import DerivParamImporter"); + StringBuilder cmd = new StringBuilder(200); + cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter("); + // Pass in directories to search based on function directories + int size = functionDirs.size() - 1; + for (int i = size; i > 0; --i) { + if (i < size) { + cmd.append(", "); + } + cmd.append("'").append(functionDirs.get(i)).append("'"); + } + cmd.append("))"); + preEvals.add(cmd.toString()); + return new MasterDerivScript(PATH, + MasterDerivScript.class.getClassLoader(), preEvals); + } + +} diff --git a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java index 5eec10a448..6c835bd384 100644 --- a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java +++ b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java @@ -20,6 +20,7 @@ package com.raytheon.uf.viz.derivparam; import java.util.List; +import java.util.concurrent.ExecutionException; import com.raytheon.uf.common.datastorage.records.IDataRecord; import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument; @@ -33,7 +34,9 @@ import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Dec 16, 2010 mschenke Initial creation + * Dec 16, 2010 mschenke Initial creation + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @@ -76,24 +79,11 @@ public interface IDerivParamFunctionAdapter { * @param arguments * arguments to pass into function */ - public void executeFunction(String name, List arguments); + public List executeFunction(String name, List arguments) + throws ExecutionException; /** - * Implementing Adapters need to be able to return the last executed - * function as data records copied into java from whatever mechanism was - * used to execute the function - * - * @return The last executed function results as data records + * Stop the adapter and any child threads. */ - public List getRequestResults(); - - /** - * Implementing Adapters need to be able to return the last executed - * function as an object that the adapter can reuse in another function - * execution. This is to avoid copying data in and out of java constantly - * - * @return The last executed function results as an object the adapter can - * reuse - */ - public Object getRequestIdentifierResults(); + public void shutdown(); } diff --git a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java index 2c7dfeb806..ceaaceba63 100644 --- a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java +++ b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import com.raytheon.uf.common.dataplugin.level.Level; import com.raytheon.uf.common.datastorage.records.FloatDataRecord; @@ -42,7 +43,9 @@ import com.raytheon.uf.viz.derivparam.tree.CubeLevel; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Mar 17, 2010 bsteffen Initial creation + * Mar 17, 2010 bsteffen Initial creation + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @@ -73,21 +76,25 @@ public class DerivedRequestableData extends AbstractRequestableData { @Override public Object getDataValue(Object arg) throws VizException { DerivedParameterRequest request = createDerparRequest(arg); - DerivedParameterGenerator.addTask(request); - List finalResult = request.getQueue(); - if (finalResult != null && finalResult.size() > 0) { - if (finalResult.size() == 1) { - return new IDataRecord[] { ((IDataRecord) finalResult.get(0)) }; - } else if (finalResult.size() == 4) { - return new IDataRecord[] { - ((FloatDataRecord) finalResult.get(0)), - ((FloatDataRecord) finalResult.get(1)), - ((FloatDataRecord) finalResult.get(2)), - ((FloatDataRecord) finalResult.get(3)) }; - } else { - throw new VizException( - "Error processing derived parameter, expecting scalar or vector data. Vector data must return speed, dir, u, and v components."); + try { + List finalResult = DerivedParameterGenerator.calculate(request); + if (finalResult != null && finalResult.size() > 0) { + if (finalResult.size() == 1) { + return new IDataRecord[] { ((IDataRecord) finalResult + .get(0)) }; + } else if (finalResult.size() == 4) { + return new IDataRecord[] { + ((FloatDataRecord) finalResult.get(0)), + ((FloatDataRecord) finalResult.get(1)), + ((FloatDataRecord) finalResult.get(2)), + ((FloatDataRecord) finalResult.get(3)) }; + } else { + throw new VizException( + "Error processing derived parameter, expecting scalar or vector data. Vector data must return speed, dir, u, and v components."); + } } + } catch (ExecutionException e) { + throw new VizException("Error executing Derived Parameter.", e); } return null; } diff --git a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java index 0695094eca..166520afe3 100644 --- a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java +++ b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java @@ -20,14 +20,15 @@ package com.raytheon.uf.viz.derivparam.library; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ExecutionException; import javax.measure.unit.Unit; import javax.xml.bind.JAXBException; @@ -77,10 +78,13 @@ import com.raytheon.uf.viz.derivparam.library.DerivParamMethod.MethodType; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Jul 3, 2008 brockwoo Initial creation - * Nov 16, 2009 #3120 rjpeter Removed use of LevelNameMappingFile. - * Nov 20, 2009 #3387 jelkins Use derived script's variableId instead of filename - * Nov 21, 2009 #3576 rjpeter Refactored DerivParamDesc. + * Jul 03, 2008 brockwoo Initial creation + * Nov 16, 2009 3120 rjpeter Removed use of LevelNameMappingFile. + * Nov 20, 2009 3387 jelkins Use derived script's variableId instead + * of filename + * Nov 21, 2009 3576 rjpeter Refactored DerivParamDesc. + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @author brockwoo @@ -113,9 +117,41 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { // TODO: Handle multiple function types (python mixed with // gsl/cuda/anything) - private static DerivParamFunctionType functionType; + private IDerivParamFunctionAdapter adapter; - static { + private Set listeners = new HashSet(); + + private Map derParLibrary; + + private boolean needsLibInit = true; + + private String extension = null; + + private Job notifyJob = new Job("Notify Derived Parameter Listeners") { + + @Override + protected IStatus run(IProgressMonitor arg0) { + Collection l = null; + synchronized (listeners) { + l = new ArrayList(listeners); + } + for (DerivParamUpdateListener listener : l) { + listener.updateDerParLibrary(derParLibrary); + } + return Status.OK_STATUS; + } + + }; + + public static synchronized DerivedParameterGenerator getInstance() { + if (instance == null) { + instance = new DerivedParameterGenerator(); + } + return instance; + } + + public static DerivParamFunctionType[] getFunctionTypes() { + List functionTypes = new ArrayList(); IExtensionRegistry registry = Platform.getExtensionRegistry(); if (registry != null) { IExtensionPoint point = registry.getExtensionPoint(EXTENSION); @@ -134,54 +170,15 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { functionType .setAdapter((IDerivParamFunctionAdapter) cfg .createExecutableExtension("adapter")); - DerivedParameterGenerator.functionType = functionType; - break; + functionTypes.add(functionType); } catch (Throwable t) { + statusHandler.handle(Priority.DEBUG, + t.getLocalizedMessage(), t); } } - if (functionType != null) { - break; - } - } - - if (functionType == null) { - statusHandler.handle(Priority.PROBLEM, - "Error creating derived parameter function type," - + " derived paramters will not be available"); } } - } - - private Set listeners = new HashSet(); - - private Map derParLibrary; - - private BlockingQueue toDoList; - - private static int numJobs = 2; - - private DerivedParameterJob[] jobs = new DerivedParameterJob[numJobs]; - - private boolean needsLibInit = true; - - private boolean runtime = true; - - private String extension = null; - - public static synchronized DerivedParameterGenerator getInstance() { - if (instance == null) { - instance = new DerivedParameterGenerator(); - } - return instance; - } - - public static DerivParamFunctionType[] getFunctionTypes() { - DerivedParameterGenerator gen = getInstance(); - if (gen != null && gen.jobs.length > 0) { - return new DerivParamFunctionType[] { instance.jobs[0].functionType }; - } - - return new DerivParamFunctionType[0]; + return functionTypes.toArray(new DerivParamFunctionType[0]); } public static synchronized Map getDerParLibrary() { @@ -189,52 +186,23 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { } public static void registerUpdateListener(DerivParamUpdateListener listener) { - getInstance().listeners.add(listener); + DerivedParameterGenerator instance = getInstance(); + synchronized (instance.listeners) { + instance.listeners.add(listener); + } } private DerivedParameterGenerator() { - for (int i = 0; i < jobs.length; i++) { - jobs[i] = new DerivedParameterJob("DerivParam Engine " + (i + 1)); + DerivParamFunctionType[] functionTypes = getFunctionTypes(); + + if (functionTypes == null || functionTypes.length == 0) { + statusHandler.handle(Priority.PROBLEM, + "Error creating derived parameter function type," + + " derived paramters will not be available"); } - - // load the extensions - IExtensionRegistry registry = Platform.getExtensionRegistry(); - if (registry != null) { - IExtensionPoint point = registry.getExtensionPoint(EXTENSION); - - IExtension[] extensions = point.getExtensions(); - - for (IExtension ext : extensions) { - IConfigurationElement[] config = ext.getConfigurationElements(); - - for (DerivedParameterJob job : jobs) { - for (IConfigurationElement cfg : config) { - try { - DerivParamFunctionType functionType = new DerivParamFunctionType(); - functionType.setName(cfg.getAttribute("name")); - functionType.setExtension(cfg - .getAttribute("extension")); - this.extension = functionType.getExtension(); - functionType - .setAdapter((IDerivParamFunctionAdapter) cfg - .createExecutableExtension("adapter")); - job.functionType = functionType; - break; - } catch (Throwable t) { - } - } - if (job.functionType == null) { - statusHandler - .handle(Priority.PROBLEM, - "Error creating derived parameter function type," - + " derived paramters will not be available"); - break; - } - } - } - } - - toDoList = new LinkedBlockingQueue(); + this.adapter = functionTypes[0].getAdapter(); + this.extension = functionTypes[0].getExtension(); + notifyJob.setSystem(true); LocalizationFile dir = PathManagerFactory.getPathManager() .getStaticLocalizationFile(DERIV_PARAM_DIR); if (dir != null) { @@ -242,9 +210,6 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { } initLibrary(); - for (int i = 0; i < jobs.length; i++) { - jobs[i].schedule(); - } } /** @@ -254,11 +219,13 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { * A derived parameter request * @return boolean indicating if the request was put into queue */ - public static boolean addTask(DerivedParameterRequest task) { - return getInstance().toDoList.add(task); + public static List calculate(DerivedParameterRequest task) + throws ExecutionException { + return getInstance().adapter.executeFunction(task.getMethod(), + Arrays.asList(task.getArgumentRecords())); } - private void initLibrary() { + private synchronized void initLibrary() { if (needsLibInit) { long start = System.currentTimeMillis(); Set derivParamFiles = new HashSet(); @@ -334,11 +301,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { } } this.derParLibrary = derParLibrary; - for (DerivParamUpdateListener listener : listeners) { - listener.updateDerParLibrary(derParLibrary); - } + adapter.init(); - System.out.println("time to init derived parameter thread: " + notifyJob.schedule(); + + System.out.println("time to init derived parameters: " + (System.currentTimeMillis() - start) + "ms"); needsLibInit = false; } @@ -359,86 +326,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver { @Override public void fileUpdated(FileUpdatedMessage message) { needsLibInit = true; - for (DerivedParameterJob job : jobs) { - job.needsInit = true; - } + initLibrary(); } public static void shutdown() { - if (instance != null) { - instance.runtime = false; - } + getInstance().adapter.shutdown(); } - private class DerivedParameterJob extends Job { - // TODO: Handle multiple function types (python mixed with - // gsl/cuda/anything) - private DerivParamFunctionType functionType; - - private boolean needsInit = true; - - /** - * @param name - */ - public DerivedParameterJob(String name) { - super(name); - this.setSystem(true); - } - - private void init() { - if (needsInit) { - functionType.getAdapter().init(); - DerivedParameterGenerator.this.initLibrary(); - needsInit = false; - } - } - - /* - * (non-Javadoc) - * - * @seeorg.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime. - * IProgressMonitor) - */ - @Override - protected IStatus run(IProgressMonitor monitor) { - while (runtime) { - if (derParLibrary == null) { - this.init(); - } - - DerivedParameterRequest request = null; - try { - request = toDoList.take(); - } catch (InterruptedException e) { - } - if (needsInit) { - init(); - } - if (request != null) { - // long t0 = System.currentTimeMillis(); - try { - functionType.getAdapter().executeFunction( - request.getMethod(), - Arrays.asList(request.getArgumentRecords())); - List result = functionType.getAdapter() - .getRequestResults(); - for (IDataRecord rec : result) { - rec.setName(request.getParameterAbbreviation()); - } - request.setQueue(result); - } catch (Throwable e) { - statusHandler - .handle(Priority.PROBLEM, - "Derived Parameter Engine failed to generate parameter", - e); - } - // System.out.println(getName() + " calc took: " - // + (System.currentTimeMillis() - t0)); - - } - } - return Status.OK_STATUS; - } - - } } diff --git a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java index ec8424c824..11ee8eb79d 100644 --- a/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java +++ b/cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java @@ -21,11 +21,8 @@ package com.raytheon.uf.viz.derivparam.library; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.TimeUnit; import com.raytheon.uf.common.time.DataTime; -import com.raytheon.uf.viz.core.exception.VizException; /** * The DerivedParameteRequest is the way for a class to communicate a request @@ -36,8 +33,10 @@ import com.raytheon.uf.viz.core.exception.VizException; * 
  * SOFTWARE HISTORY
  * Date			Ticket#		Engineer	Description
- * ------------	----------	-----------	--------------------------
- * Jul 9, 2008				brockwoo	Initial creation
+ * ------------ ----------  ----------- --------------------------
+ * Jul 09, 2008             brockwoo    Initial creation
+ * Jun 04, 2013 2041        bsteffen    Switch derived parameters to use
+ *                                      concurrent python for threading.
  * 
  *
* @@ -56,10 +55,7 @@ public class DerivedParameterRequest { private DataTime baseTime; - private SynchronousQueue> queue; - public DerivedParameterRequest() { - this.queue = new SynchronousQueue>(); this.baseParam = new ArrayList(); } @@ -164,41 +160,4 @@ public class DerivedParameterRequest { this.argumentRecords = argumentRecords; } - /** - * This method will return the data calculated by the derived parameter - * script. It is important to call this method right after the request has - * been submitted to the derived parameter thread as it will block for 30 - * seconds waiting for the response. - * - * @return the raw data calculated by the script - */ - public List getQueue() throws VizException { - try { - List result = queue.poll(30, TimeUnit.SECONDS); - if (result == null) { - throw new VizException("Derived Parameter Timed Out"); - } - return result; - } catch (InterruptedException e) { - throw new VizException(e); - } - } - - /** - * Used by the derived parameter generator to return the calculated data to - * the thread requesting it. This will block until getQueue is - * called. - * - * @param queue - * the data being returned to the thread making the request - * @throws VizException - */ - public void setQueue(List queue) { - try { - this.queue.offer(queue, 15, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } diff --git a/cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java b/cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java index 832e39a6e2..e9daeccdbe 100644 --- a/cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java +++ b/cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java @@ -22,6 +22,7 @@ package com.raytheon.uf.viz.objectiveanalysis.rsc; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import org.geotools.coverage.grid.GeneralGridGeometry; import org.opengis.referencing.crs.CoordinateReferenceSystem; @@ -53,7 +54,9 @@ import com.raytheon.viz.grid.util.TiltUtils; * SOFTWARE HISTORY * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * May 20, 2010 bsteffen Initial creation + * May 20, 2010 bsteffen Initial creation + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @@ -210,9 +213,12 @@ public class OATiltGridTransformer extends OAGridTransformer { DerivedParameterRequest sliceRequest = new DerivedParameterRequest(); sliceRequest.setMethod("Slice"); sliceRequest.setArgumentRecords(new Object[] { cube, presRequest, -1 }); - DerivedParameterGenerator.addTask(sliceRequest); - return ((FloatDataRecord) sliceRequest.getQueue().get(0)) - .getFloatData(); + try { + return ((FloatDataRecord) DerivedParameterGenerator.calculate( + sliceRequest).get(0)).getFloatData(); + } catch (ExecutionException e) { + throw new VizException(e); + } } diff --git a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java index c8c106ea34..82cc4783ce 100644 --- a/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java +++ b/cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import javax.media.jai.Interpolation; @@ -84,11 +85,12 @@ import com.raytheon.uf.viz.derivparam.library.IDerivParamField; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jul 27, 2009 jsanchez Initial creation - * Nov 21, 2009 #3576 rjpeter Refactored use of DerivParamDesc. - * - AWIPS2 Baseline Repository -------- - * 08/03/2012 798 jkorman Explicitly set interpolationLevels - * from "source" record. - * 04/08/2013 #1293 bkowal Removed references to hdffileid. + * Nov 21, 2009 3576 rjpeter Refactored use of DerivParamDesc. + * Aug 03, 2012 798 jkorman Explicitly set interpolationLevels from + * "source" record. + * Apr 08, 2013 1293 bkowal Removed references to hdffileid. + * Jun 04, 2013 2041 bsteffen Switch derived parameters to use + * concurrent python for threading. * * * @author jsanchez @@ -313,11 +315,10 @@ public class SatelliteDataCubeAdapter implements IDataCubeAdapter { } derivedRequest.setArgumentRecords(bytes.toArray(new Object[] {})); - DerivedParameterGenerator.addTask(derivedRequest); List finalResult; try { - finalResult = derivedRequest.getQueue(); - } catch (VizException e) { + finalResult = DerivedParameterGenerator.calculate(derivedRequest); + } catch (ExecutionException e) { throw new VizDataCubeException(e); }