Issue #2041 Switch derived parameters to use concurrent python for threading.

Change-Id: I791eb2f4803b5d29928dd153bbcf157ae643534f

Former-commit-id: 11f346d4d7baae08f61966a91e4d5f579630c7de

cave/com.raytheon.uf.viz.derivparam.python/META-INF/MANIFEST.MF

@@ -17,6 +17,7 @@ Require-Bundle: com.raytheon.uf.common.status;bundle-version="1.12.1174",
  org.jep;bundle-version="1.0.0",
  org.eclipse.ui;bundle-version="3.6.1",
  org.eclipse.core.runtime;bundle-version="3.6.0",
- com.raytheon.uf.common.util;bundle-version="1.12.1174"
+ com.raytheon.uf.common.util;bundle-version="1.12.1174",
+ com.raytheon.uf.common.python.concurrent;bundle-version="1.0.0"
 Export-Package: com.raytheon.uf.viz.derivparam.python,
  com.raytheon.uf.viz.derivparam.python.function

cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/DerivParamPythonFunctionAdapter.java

@@ -22,20 +22,13 @@ package com.raytheon.uf.viz.derivparam.python;
 import java.io.File;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
-
+import java.util.concurrent.ExecutionException;
-import jep.JepException;
 
 import com.raytheon.uf.common.datastorage.records.IDataRecord;
-import com.raytheon.uf.common.localization.IPathManager;
-import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
-import com.raytheon.uf.common.localization.LocalizationFile;
-import com.raytheon.uf.common.localization.LocalizationUtil;
 import com.raytheon.uf.common.localization.PathManagerFactory;
-import com.raytheon.uf.common.python.PyUtil;
+import com.raytheon.uf.common.python.concurrent.PythonJobCoordinator;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
@@ -53,7 +46,9 @@ import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator;
  * 
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Dec 16, 2010            mschenke     Initial creation
+ * Dec 16, 2010            mschenke    Initial creation
+ * Jun 04, 2013 2041       bsteffen    Switch derived parameters to use
+ *                                     concurrent python for threading.
  * 
  * </pre>
  * 
@@ -68,18 +63,10 @@ public class DerivParamPythonFunctionAdapter implements
 
     private static final String PYTHON = "python";
 
-    private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR
-            + File.separator
-            + PYTHON
-            + File.separator
-            + "DerivParamImporter.py";
-
     private static final String TEMPLATE_FILE = DerivedParameterGenerator.DERIV_PARAM_DIR
             + File.separator + PYTHON + File.separator + "functionTemplate.txt";
 
-    private MasterDerivScript masterScript;
+    private PythonJobCoordinator<MasterDerivScript> coordinator;
-
-    private List<IDataRecord> results;
 
     /*
      * (non-Javadoc)
@@ -134,58 +121,11 @@ public class DerivParamPythonFunctionAdapter implements
      */
     @Override
     public void init() {
-        IPathManager pm = PathManagerFactory.getPathManager();
+        if (coordinator != null) {
-
+            coordinator.shutdown();
-        File script = pm.getStaticFile(INTERFACE_SCRIPT);
-
-        // Get list of all files for search hierarch of CAVE_STATIC
-        LocalizationFile[] derivParamFiles = pm.listFiles(
-                pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC),
-                DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false);
-        List<String> functionDirs = new ArrayList<String>(
-                derivParamFiles.length);
-        functionDirs.add(script.getParent());
-
-        Arrays.sort(derivParamFiles);
-
-        for (LocalizationFile file : derivParamFiles) {
-            if (file.isDirectory()
-                    && DerivedParameterGenerator.FUNCTIONS
-                            .equals(LocalizationUtil.extractName(file.getName()))) {
-                // If it is a derived parameters functions directory, add to search list
-                functionDirs.add(file.getFile().getAbsolutePath());
-            }
-        }
-
-        // Create path from function dir list
-        String PATH = PyUtil.buildJepIncludePath(functionDirs
-                .toArray(new String[functionDirs.size()]));
-
-        List<String> preEvals = new ArrayList<String>(2);
-        preEvals.add("import DerivParamImporter");
-        StringBuilder cmd = new StringBuilder(200);
-        cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter(");
-        // Pass in directories to search based on function directories
-        int size = functionDirs.size() - 1;
-        for (int i = size; i > 0; --i) {
-            if (i < size) {
-                cmd.append(", ");
-            }
-            cmd.append("'").append(functionDirs.get(i)).append("'");
-        }
-        cmd.append("))");
-        preEvals.add(cmd.toString());
-
-        try {
-            if (masterScript != null) {
-                masterScript.dispose();
-            }
-            masterScript = new MasterDerivScript(PATH,
-                    MasterDerivScript.class.getClassLoader(), preEvals);
-        } catch (JepException e) {
-            statusHandler.handle(Priority.PROBLEM,
-                    "Failed to load derived parameters", e);
         }
+        coordinator = PythonJobCoordinator
+                .newInstance(new MasterDerivScriptFactory());
     }
 
     /*
@@ -195,41 +135,27 @@ public class DerivParamPythonFunctionAdapter implements
      * com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#executeFunction
      * (java.lang.String, java.util.List)
      */
-    @SuppressWarnings("unchecked")
     @Override
-    public void executeFunction(String name, List<Object> arguments) {
+    public List<IDataRecord> executeFunction(String name, List<Object> arguments)
+            throws ExecutionException {
         try {
-            results = (List<IDataRecord>) masterScript.executeFunction(name,
+            return coordinator.submitSyncJob(new MasterDerivScriptExecutor(
-                    arguments);
+                    name, arguments));
-        } catch (JepException e) {
+        } catch (InterruptedException e) {
-            statusHandler.handle(Priority.PROBLEM,
+            throw new ExecutionException(e);
-                    "Error executing derived parameter request", e);
-            results = null;
         }
     }
 
     /*
      * (non-Javadoc)
      * 
-     * @see
+     * @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#shutdown()
-     * com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#getRequestResults
-     * ()
      */
     @Override
-    public List<IDataRecord> getRequestResults() {
+    public void shutdown() {
-        return results;
+        if (coordinator != null) {
-    }
+            coordinator.shutdown();
-
+        }
-    /*
-     * (non-Javadoc)
-     * 
-     * @see com.raytheon.uf.viz.derivparam.IDerivParamFunctionAdapter#
-     * getRequestIdentifierResults()
-     */
-    @Override
-    public Object getRequestIdentifierResults() {
-        // TODO: Not supported yet
-        return null;
     }
 
 }

cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptExecutor.java

@@ -0,0 +1,65 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.viz.derivparam.python;
+
+import java.util.List;
+
+import jep.JepException;
+
+import com.raytheon.uf.common.datastorage.records.IDataRecord;
+import com.raytheon.uf.common.python.concurrent.IPythonExecutor;
+
+/**
+ * Executor for calling executeFunction on a MasterDerivScript
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 04, 2013 2041       bsteffen    Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+
+public class MasterDerivScriptExecutor implements
+        IPythonExecutor<MasterDerivScript, List<IDataRecord>> {
+
+    private final String name;
+
+    private final List<Object> arguments;
+
+    public MasterDerivScriptExecutor(String name, List<Object> arguments) {
+        this.name = name;
+        this.arguments = arguments;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public List<IDataRecord> execute(MasterDerivScript script)
+            throws JepException {
+        return (List<IDataRecord>) script.executeFunction(name, arguments);
+    }
+
+}

cave/com.raytheon.uf.viz.derivparam.python/src/com/raytheon/uf/viz/derivparam/python/MasterDerivScriptFactory.java

@@ -0,0 +1,120 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.viz.derivparam.python;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import jep.JepException;
+
+import com.raytheon.uf.common.localization.IPathManager;
+import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
+import com.raytheon.uf.common.localization.LocalizationFile;
+import com.raytheon.uf.common.localization.LocalizationUtil;
+import com.raytheon.uf.common.localization.PathManagerFactory;
+import com.raytheon.uf.common.python.PyUtil;
+import com.raytheon.uf.common.python.concurrent.AbstractPythonScriptFactory;
+import com.raytheon.uf.viz.derivparam.library.DerivedParameterGenerator;
+
+/**
+ * Factory for creating and initializing MasterDerivScript.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jun 04, 2013 2041       bsteffen    Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+
+public class MasterDerivScriptFactory extends
+        AbstractPythonScriptFactory<MasterDerivScript> {
+
+    private static final int MAX_THREADS = 2;
+
+    public static final String NAME = "DerivedParameterPython";
+
+    private static final String INTERFACE_SCRIPT = DerivedParameterGenerator.DERIV_PARAM_DIR
+            + File.separator
+            + "python"
+            + File.separator
+            + "DerivParamImporter.py";
+
+    public MasterDerivScriptFactory() {
+        super(NAME, MAX_THREADS);
+    }
+
+    @Override
+    public MasterDerivScript createPythonScript() throws JepException {
+        IPathManager pm = PathManagerFactory.getPathManager();
+
+        File script = pm.getStaticFile(INTERFACE_SCRIPT);
+
+        // Get list of all files for search hierarch of CAVE_STATIC
+        LocalizationFile[] derivParamFiles = pm.listFiles(
+                pm.getLocalSearchHierarchy(LocalizationType.CAVE_STATIC),
+                DerivedParameterGenerator.DERIV_PARAM_DIR, null, false, false);
+        List<String> functionDirs = new ArrayList<String>(
+                derivParamFiles.length);
+        functionDirs.add(script.getParent());
+
+        Arrays.sort(derivParamFiles);
+
+        for (LocalizationFile file : derivParamFiles) {
+            if (file.isDirectory()
+                    && DerivedParameterGenerator.FUNCTIONS
+                            .equals(LocalizationUtil.extractName(file.getName()))) {
+                // If it is a derived parameters functions directory, add to
+                // search list
+                functionDirs.add(file.getFile().getAbsolutePath());
+            }
+        }
+
+        // Create path from function dir list
+        String PATH = PyUtil.buildJepIncludePath(functionDirs
+                .toArray(new String[functionDirs.size()]));
+
+        List<String> preEvals = new ArrayList<String>(2);
+        preEvals.add("import DerivParamImporter");
+        StringBuilder cmd = new StringBuilder(200);
+        cmd.append("sys.meta_path.append(DerivParamImporter.DerivParamImporter(");
+        // Pass in directories to search based on function directories
+        int size = functionDirs.size() - 1;
+        for (int i = size; i > 0; --i) {
+            if (i < size) {
+                cmd.append(", ");
+            }
+            cmd.append("'").append(functionDirs.get(i)).append("'");
+        }
+        cmd.append("))");
+        preEvals.add(cmd.toString());
+        return new MasterDerivScript(PATH,
+                MasterDerivScript.class.getClassLoader(), preEvals);
+    }
+
+}

cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/IDerivParamFunctionAdapter.java

@@ -20,6 +20,7 @@
 package com.raytheon.uf.viz.derivparam;
 
 import java.util.List;
+import java.util.concurrent.ExecutionException;
 
 import com.raytheon.uf.common.datastorage.records.IDataRecord;
 import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument;
@@ -33,7 +34,9 @@ import com.raytheon.uf.viz.derivparam.DerivParamFunctionType.FunctionArgument;
  * 
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Dec 16, 2010            mschenke     Initial creation
+ * Dec 16, 2010            mschenke    Initial creation
+ * Jun 04, 2013 2041       bsteffen    Switch derived parameters to use
+ *                                     concurrent python for threading.
  * 
  * </pre>
  * 
@@ -76,24 +79,11 @@ public interface IDerivParamFunctionAdapter {
      * @param arguments
      *            arguments to pass into function
      */
-    public void executeFunction(String name, List<Object> arguments);
+    public List<IDataRecord> executeFunction(String name, List<Object> arguments)
+            throws ExecutionException;
 
     /**
-     * Implementing Adapters need to be able to return the last executed
+     * Stop the adapter and any child threads.
-     * function as data records copied into java from whatever mechanism was
-     * used to execute the function
-     * 
-     * @return The last executed function results as data records
      */
-    public List<IDataRecord> getRequestResults();
+    public void shutdown();
-
-    /**
-     * Implementing Adapters need to be able to return the last executed
-     * function as an object that the adapter can reuse in another function
-     * execution. This is to avoid copying data in and out of java constantly
-     * 
-     * @return The last executed function results as an object the adapter can
-     *         reuse
-     */
-    public Object getRequestIdentifierResults();
 }

cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/data/DerivedRequestableData.java

@@ -25,6 +25,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 import com.raytheon.uf.common.dataplugin.level.Level;
 import com.raytheon.uf.common.datastorage.records.FloatDataRecord;
@@ -42,7 +43,9 @@ import com.raytheon.uf.viz.derivparam.tree.CubeLevel;
  * SOFTWARE HISTORY
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Mar 17, 2010            bsteffen     Initial creation
+ * Mar 17, 2010            bsteffen    Initial creation
+ * Jun 04, 2013 2041       bsteffen    Switch derived parameters to use
+ *                                     concurrent python for threading.
  * 
  * </pre>
  * 
@@ -73,21 +76,25 @@ public class DerivedRequestableData extends AbstractRequestableData {
     @Override
     public Object getDataValue(Object arg) throws VizException {
         DerivedParameterRequest request = createDerparRequest(arg);
-        DerivedParameterGenerator.addTask(request);
+        try {
-        List<?> finalResult = request.getQueue();
+            List<?> finalResult = DerivedParameterGenerator.calculate(request);
-        if (finalResult != null && finalResult.size() > 0) {
+            if (finalResult != null && finalResult.size() > 0) {
-            if (finalResult.size() == 1) {
+                if (finalResult.size() == 1) {
-                return new IDataRecord[] { ((IDataRecord) finalResult.get(0)) };
+                    return new IDataRecord[] { ((IDataRecord) finalResult
-            } else if (finalResult.size() == 4) {
+                            .get(0)) };
-                return new IDataRecord[] {
+                } else if (finalResult.size() == 4) {
-                        ((FloatDataRecord) finalResult.get(0)),
+                    return new IDataRecord[] {
-                        ((FloatDataRecord) finalResult.get(1)),
+                            ((FloatDataRecord) finalResult.get(0)),
-                        ((FloatDataRecord) finalResult.get(2)),
+                            ((FloatDataRecord) finalResult.get(1)),
-                        ((FloatDataRecord) finalResult.get(3)) };
+                            ((FloatDataRecord) finalResult.get(2)),
-            } else {
+                            ((FloatDataRecord) finalResult.get(3)) };
-                throw new VizException(
+                } else {
-                        "Error processing derived parameter, expecting scalar or vector data.  Vector data must return speed, dir, u, and v components.");
+                    throw new VizException(
+                            "Error processing derived parameter, expecting scalar or vector data.  Vector data must return speed, dir, u, and v components.");
+                }
             }
+        } catch (ExecutionException e) {
+            throw new VizException("Error executing Derived Parameter.", e);
         }
         return null;
     }

cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterGenerator.java

@@ -20,14 +20,15 @@
 package com.raytheon.uf.viz.derivparam.library;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutionException;
-import java.util.concurrent.LinkedBlockingQueue;
 
 import javax.measure.unit.Unit;
 import javax.xml.bind.JAXBException;
@@ -77,10 +78,13 @@ import com.raytheon.uf.viz.derivparam.library.DerivParamMethod.MethodType;
  * SOFTWARE HISTORY
  * Date         Ticket#     Engineer    Description
  * ------------ ----------  ----------- --------------------------
- * Jul 3, 2008              brockwoo    Initial creation
+ * Jul 03, 2008             brockwoo    Initial creation
- * Nov 16, 2009 #3120       rjpeter     Removed use of LevelNameMappingFile.
+ * Nov 16, 2009 3120        rjpeter     Removed use of LevelNameMappingFile.
- * Nov 20, 2009 #3387       jelkins     Use derived script's variableId instead of filename
+ * Nov 20, 2009 3387        jelkins     Use derived script's variableId instead
- * Nov 21, 2009 #3576       rjpeter     Refactored DerivParamDesc.
+ *                                      of filename
+ * Nov 21, 2009 3576        rjpeter     Refactored DerivParamDesc.
+ * Jun 04, 2013 2041        bsteffen    Switch derived parameters to use
+ *                                      concurrent python for threading.
  * </pre>
  * 
  * @author brockwoo
@@ -113,9 +117,41 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
 
     // TODO: Handle multiple function types (python mixed with
     // gsl/cuda/anything)
-    private static DerivParamFunctionType functionType;
+    private IDerivParamFunctionAdapter adapter;
 
-    static {
+    private Set<DerivParamUpdateListener> listeners = new HashSet<DerivParamUpdateListener>();
+
+    private Map<String, DerivParamDesc> derParLibrary;
+
+    private boolean needsLibInit = true;
+
+    private String extension = null;
+
+    private Job notifyJob = new Job("Notify Derived Parameter Listeners") {
+        
+        @Override
+        protected IStatus run(IProgressMonitor arg0) {
+            Collection<DerivParamUpdateListener> l = null;
+            synchronized (listeners) {
+                l = new ArrayList<DerivParamUpdateListener>(listeners);
+            }
+            for (DerivParamUpdateListener listener : l) {
+                listener.updateDerParLibrary(derParLibrary);
+            }
+            return Status.OK_STATUS;
+        }
+
+    };
+    
+    public static synchronized DerivedParameterGenerator getInstance() {
+        if (instance == null) {
+            instance = new DerivedParameterGenerator();
+        }
+        return instance;
+    }
+
+    public static DerivParamFunctionType[] getFunctionTypes() {
+        List<DerivParamFunctionType> functionTypes = new ArrayList<DerivParamFunctionType>();
         IExtensionRegistry registry = Platform.getExtensionRegistry();
         if (registry != null) {
             IExtensionPoint point = registry.getExtensionPoint(EXTENSION);
@@ -134,54 +170,15 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
                         functionType
                                 .setAdapter((IDerivParamFunctionAdapter) cfg
                                         .createExecutableExtension("adapter"));
-                        DerivedParameterGenerator.functionType = functionType;
+                        functionTypes.add(functionType);
-                        break;
                     } catch (Throwable t) {
+                        statusHandler.handle(Priority.DEBUG,
+                                t.getLocalizedMessage(), t);
                     }
                 }
-                if (functionType != null) {
-                    break;
-                }
-            }
-
-            if (functionType == null) {
-                statusHandler.handle(Priority.PROBLEM,
-                        "Error creating derived parameter function type,"
-                                + " derived paramters will not be available");
             }
         }
-    }
+        return functionTypes.toArray(new DerivParamFunctionType[0]);
-
-    private Set<DerivParamUpdateListener> listeners = new HashSet<DerivParamUpdateListener>();
-
-    private Map<String, DerivParamDesc> derParLibrary;
-
-    private BlockingQueue<DerivedParameterRequest> toDoList;
-
-    private static int numJobs = 2;
-
-    private DerivedParameterJob[] jobs = new DerivedParameterJob[numJobs];
-
-    private boolean needsLibInit = true;
-
-    private boolean runtime = true;
-
-    private String extension = null;
-
-    public static synchronized DerivedParameterGenerator getInstance() {
-        if (instance == null) {
-            instance = new DerivedParameterGenerator();
-        }
-        return instance;
-    }
-
-    public static DerivParamFunctionType[] getFunctionTypes() {
-        DerivedParameterGenerator gen = getInstance();
-        if (gen != null && gen.jobs.length > 0) {
-            return new DerivParamFunctionType[] { instance.jobs[0].functionType };
-        }
-
-        return new DerivParamFunctionType[0];
     }
 
     public static synchronized Map<String, DerivParamDesc> getDerParLibrary() {
@@ -189,52 +186,23 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
     }
 
     public static void registerUpdateListener(DerivParamUpdateListener listener) {
-        getInstance().listeners.add(listener);
+        DerivedParameterGenerator instance = getInstance();
+        synchronized (instance.listeners) {
+            instance.listeners.add(listener);
+        }
     }
 
     private DerivedParameterGenerator() {
-        for (int i = 0; i < jobs.length; i++) {
+        DerivParamFunctionType[] functionTypes = getFunctionTypes();
-            jobs[i] = new DerivedParameterJob("DerivParam Engine " + (i + 1));
+
+        if (functionTypes == null || functionTypes.length == 0) {
+            statusHandler.handle(Priority.PROBLEM,
+                    "Error creating derived parameter function type,"
+                            + " derived paramters will not be available");
         }
-
+        this.adapter = functionTypes[0].getAdapter();
-        // load the extensions
+        this.extension = functionTypes[0].getExtension();
-        IExtensionRegistry registry = Platform.getExtensionRegistry();
+        notifyJob.setSystem(true);
-        if (registry != null) {
-            IExtensionPoint point = registry.getExtensionPoint(EXTENSION);
-
-            IExtension[] extensions = point.getExtensions();
-
-            for (IExtension ext : extensions) {
-                IConfigurationElement[] config = ext.getConfigurationElements();
-
-                for (DerivedParameterJob job : jobs) {
-                    for (IConfigurationElement cfg : config) {
-                        try {
-                            DerivParamFunctionType functionType = new DerivParamFunctionType();
-                            functionType.setName(cfg.getAttribute("name"));
-                            functionType.setExtension(cfg
-                                    .getAttribute("extension"));
-                            this.extension = functionType.getExtension();
-                            functionType
-                                    .setAdapter((IDerivParamFunctionAdapter) cfg
-                                            .createExecutableExtension("adapter"));
-                            job.functionType = functionType;
-                            break;
-                        } catch (Throwable t) {
-                        }
-                    }
-                    if (job.functionType == null) {
-                        statusHandler
-                                .handle(Priority.PROBLEM,
-                                        "Error creating derived parameter function type,"
-                                                + " derived paramters will not be available");
-                        break;
-                    }
-                }
-            }
-        }
-
-        toDoList = new LinkedBlockingQueue<DerivedParameterRequest>();
         LocalizationFile dir = PathManagerFactory.getPathManager()
                 .getStaticLocalizationFile(DERIV_PARAM_DIR);
         if (dir != null) {
@@ -242,9 +210,6 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
         }
 
         initLibrary();
-        for (int i = 0; i < jobs.length; i++) {
-            jobs[i].schedule();
-        }
     }
 
     /**
@@ -254,11 +219,13 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
      *            A derived parameter request
      * @return boolean indicating if the request was put into queue
      */
-    public static boolean addTask(DerivedParameterRequest task) {
+    public static List<IDataRecord> calculate(DerivedParameterRequest task)
-        return getInstance().toDoList.add(task);
+            throws ExecutionException {
+        return getInstance().adapter.executeFunction(task.getMethod(),
+                Arrays.asList(task.getArgumentRecords()));
     }
 
-    private void initLibrary() {
+    private synchronized void initLibrary() {
         if (needsLibInit) {
             long start = System.currentTimeMillis();
             Set<String> derivParamFiles = new HashSet<String>();
@@ -334,11 +301,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
                 }
             }
             this.derParLibrary = derParLibrary;
-            for (DerivParamUpdateListener listener : listeners) {
+            adapter.init();
-                listener.updateDerParLibrary(derParLibrary);
-            }
 
-            System.out.println("time to init derived parameter thread: "
+            notifyJob.schedule();
+
+            System.out.println("time to init derived parameters: "
                     + (System.currentTimeMillis() - start) + "ms");
             needsLibInit = false;
         }
@@ -359,86 +326,11 @@ public class DerivedParameterGenerator implements ILocalizationFileObserver {
     @Override
     public void fileUpdated(FileUpdatedMessage message) {
         needsLibInit = true;
-        for (DerivedParameterJob job : jobs) {
+        initLibrary();
-            job.needsInit = true;
-        }
     }
 
     public static void shutdown() {
-        if (instance != null) {
+        getInstance().adapter.shutdown();
-            instance.runtime = false;
-        }
     }
 
-    private class DerivedParameterJob extends Job {
-        // TODO: Handle multiple function types (python mixed with
-        // gsl/cuda/anything)
-        private DerivParamFunctionType functionType;
-
-        private boolean needsInit = true;
-
-        /**
-         * @param name
-         */
-        public DerivedParameterJob(String name) {
-            super(name);
-            this.setSystem(true);
-        }
-
-        private void init() {
-            if (needsInit) {
-                functionType.getAdapter().init();
-                DerivedParameterGenerator.this.initLibrary();
-                needsInit = false;
-            }
-        }
-
-        /*
-         * (non-Javadoc)
-         * 
-         * @seeorg.eclipse.core.runtime.jobs.Job#run(org.eclipse.core.runtime.
-         * IProgressMonitor)
-         */
-        @Override
-        protected IStatus run(IProgressMonitor monitor) {
-            while (runtime) {
-                if (derParLibrary == null) {
-                    this.init();
-                }
-
-                DerivedParameterRequest request = null;
-                try {
-                    request = toDoList.take();
-                } catch (InterruptedException e) {
-                }
-                if (needsInit) {
-                    init();
-                }
-                if (request != null) {
-                    // long t0 = System.currentTimeMillis();
-                    try {
-                        functionType.getAdapter().executeFunction(
-                                request.getMethod(),
-                                Arrays.asList(request.getArgumentRecords()));
-                        List<IDataRecord> result = functionType.getAdapter()
-                                .getRequestResults();
-                        for (IDataRecord rec : result) {
-                            rec.setName(request.getParameterAbbreviation());
-                        }
-                        request.setQueue(result);
-                    } catch (Throwable e) {
-                        statusHandler
-                                .handle(Priority.PROBLEM,
-                                        "Derived Parameter Engine failed to generate parameter",
-                                        e);
-                    }
-                    // System.out.println(getName() + " calc took: "
-                    // + (System.currentTimeMillis() - t0));
-
-                }
-            }
-            return Status.OK_STATUS;
-        }
-
-    }
 }

cave/com.raytheon.uf.viz.derivparam/src/com/raytheon/uf/viz/derivparam/library/DerivedParameterRequest.java

@@ -21,11 +21,8 @@ package com.raytheon.uf.viz.derivparam.library;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
 
 import com.raytheon.uf.common.time.DataTime;
-import com.raytheon.uf.viz.core.exception.VizException;
 
 /**
  * The DerivedParameteRequest is the way for a class to communicate a request
@@ -36,8 +33,10 @@ import com.raytheon.uf.viz.core.exception.VizException;
  * <pre>
  * SOFTWARE HISTORY
  * Date			Ticket#		Engineer	Description
- * ------------	----------	-----------	--------------------------
+ * ------------ ----------  ----------- --------------------------
- * Jul 9, 2008				brockwoo	Initial creation
+ * Jul 09, 2008             brockwoo    Initial creation
+ * Jun 04, 2013 2041        bsteffen    Switch derived parameters to use
+ *                                      concurrent python for threading.
  * 
  * </pre>
  * 
@@ -56,10 +55,7 @@ public class DerivedParameterRequest {
 
     private DataTime baseTime;
 
-    private SynchronousQueue<List<?>> queue;
-
     public DerivedParameterRequest() {
-        this.queue = new SynchronousQueue<List<?>>();
         this.baseParam = new ArrayList<Object>();
     }
 
@@ -164,41 +160,4 @@ public class DerivedParameterRequest {
         this.argumentRecords = argumentRecords;
     }
 
-    /**
-     * This method will return the data calculated by the derived parameter
-     * script. It is important to call this method right after the request has
-     * been submitted to the derived parameter thread as it will block for 30
-     * seconds waiting for the response.
-     * 
-     * @return the raw data calculated by the script
-     */
-    public List<?> getQueue() throws VizException {
-        try {
-            List<?> result = queue.poll(30, TimeUnit.SECONDS);
-            if (result == null) {
-                throw new VizException("Derived Parameter Timed Out");
-            }
-            return result;
-        } catch (InterruptedException e) {
-            throw new VizException(e);
-        }
-    }
-
-    /**
-     * Used by the derived parameter generator to return the calculated data to
-     * the thread requesting it. This will block until <code>getQueue</code> is
-     * called.
-     * 
-     * @param queue
-     *            the data being returned to the thread making the request
-     * @throws VizException
-     */
-    public void setQueue(List<?> queue) {
-        try {
-            this.queue.offer(queue, 15, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            e.printStackTrace();
-        }
-    }
-
 }

cave/com.raytheon.uf.viz.objectiveanalysis/src/com/raytheon/uf/viz/objectiveanalysis/rsc/OATiltGridTransformer.java

@@ -22,6 +22,7 @@ package com.raytheon.uf.viz.objectiveanalysis.rsc;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 import org.geotools.coverage.grid.GeneralGridGeometry;
 import org.opengis.referencing.crs.CoordinateReferenceSystem;
@@ -53,7 +54,9 @@ import com.raytheon.viz.grid.util.TiltUtils;
  * SOFTWARE HISTORY
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * May 20, 2010            bsteffen     Initial creation
+ * May 20, 2010            bsteffen    Initial creation
+ * Jun 04, 2013 2041       bsteffen    Switch derived parameters to use
+ *                                     concurrent python for threading.
  * 
  * </pre>
  * 
@@ -210,9 +213,12 @@ public class OATiltGridTransformer extends OAGridTransformer {
         DerivedParameterRequest sliceRequest = new DerivedParameterRequest();
         sliceRequest.setMethod("Slice");
         sliceRequest.setArgumentRecords(new Object[] { cube, presRequest, -1 });
-        DerivedParameterGenerator.addTask(sliceRequest);
+        try {
-        return ((FloatDataRecord) sliceRequest.getQueue().get(0))
+            return ((FloatDataRecord) DerivedParameterGenerator.calculate(
-                .getFloatData();
+                    sliceRequest).get(0)).getFloatData();
+        } catch (ExecutionException e) {
+            throw new VizException(e);
+        }
 
     }
 

cave/com.raytheon.viz.satellite/src/com/raytheon/viz/satellite/SatelliteDataCubeAdapter.java

@@ -35,6 +35,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
 
 import javax.media.jai.Interpolation;
 
@@ -84,11 +85,12 @@ import com.raytheon.uf.viz.derivparam.library.IDerivParamField;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jul 27, 2009            jsanchez    Initial creation
- * Nov 21, 2009 #3576      rjpeter     Refactored use of DerivParamDesc.
+ * Nov 21, 2009 3576       rjpeter     Refactored use of DerivParamDesc.
- * - AWIPS2 Baseline Repository --------
+ * Aug 03, 2012 798        jkorman     Explicitly set interpolationLevels  from
- * 08/03/2012          798 jkorman     Explicitly set interpolationLevels
+ *                                     "source" record.
- *                                     from "source" record.
+ * Apr 08, 2013 1293       bkowal      Removed references to hdffileid.
- * 04/08/2013   #1293      bkowal      Removed references to hdffileid.    
+ * Jun 04, 2013 2041       bsteffen    Switch derived parameters to use
+ *                                     concurrent python for threading.
  * </pre>
  * 
  * @author jsanchez
@@ -313,11 +315,10 @@ public class SatelliteDataCubeAdapter implements IDataCubeAdapter {
         }
 
         derivedRequest.setArgumentRecords(bytes.toArray(new Object[] {}));
-        DerivedParameterGenerator.addTask(derivedRequest);
         List<?> finalResult;
         try {
-            finalResult = derivedRequest.getQueue();
+            finalResult = DerivedParameterGenerator.calculate(derivedRequest);
-        } catch (VizException e) {
+        } catch (ExecutionException e) {
             throw new VizDataCubeException(e);
         }