Omaha #4234 initialize the main python interpreter on a different thread

than the sub-interpreters, or warn when that happens

Change-Id: Id4d2c2401c96f6d884e5c468419f2d5893f6eafd

Former-commit-id: 5386e7fabcfbac366bb5335f601bcd0719b4f32c

build/deploy.edex.awips2/esb/conf/wrapper.conf

@@ -139,7 +139,7 @@ wrapper.java.additional.log.5=-Djava.util.logging.config.file=${EDEX_HOME}/conf/
 # errors reported by thrift sometimes when the stream is corrupt/incorrect
 wrapper.java.additional.thrift.maxStreamSize=-Dthrift.stream.maxsize=200
 
-wrapper.java.additional.retain.failed=-Dretain.failed.data=${RETAIN_FAILED}
+#wrapper.java.additional.retain.failed=-Dretain.failed.data=${RETAIN_FAILED}
 
 # enables yourkit profiling, determined by flag to start.sh
 wrapper.java.additional.profile.1=${PROFILER_PARAM_1}

edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml

@@ -8,7 +8,8 @@
         <constructor-arg ref="fetchATSrv" />
     </bean>
     
-    <bean id="gfeSitesActiveRequest" factory-bean="siteAwareRegistry" factory-method="register">
+    <bean id="gfeSitesActiveRequest" factory-bean="siteAwareRegistry" factory-method="register"
+            depends-on="jepInit">
         <constructor-arg ref="gfeSiteActivation"/>
     </bean>
 

edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-spring.xml

@@ -12,7 +12,8 @@
 
     <bean id="smartInitQueue" class="com.raytheon.edex.plugin.gfe.smartinit.SmartInitQueue" factory-method="createQueue"/>
 
-    <bean id="gfeSitesActiveIngest" factory-bean="siteAwareRegistry" factory-method="register" depends-on="smartInitQueue">
+    <bean id="gfeSitesActiveIngest" factory-bean="siteAwareRegistry" factory-method="register"
+            depends-on="jepInit, smartInitQueue">
         <constructor-arg ref="gfeSiteActivation"/>
     </bean>
 

edexOsgi/com.raytheon.uf.edex.plugin.grid/modes/grid-modes.xml

@@ -31,6 +31,7 @@
 	<mode name="ingestGrib">
         <include>time-common.xml</include>
         <include>auth-common.xml</include>
+        <include>python-common.xml</include>
         <include>grib-decode.xml</include>
         <include>ncgrib-file-endpoint.xml</include>
         <include>grid-staticdata-process.xml</include>

edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-common.xml

@@ -3,7 +3,8 @@
 	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
 
-	<bean id="qcProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
+	<bean id="qcProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties"
+            depends-on="jepInit">
 		<property name="pluginName" value="qc" />
         <property name="pluginFQN" value="com.raytheon.uf.common.dataplugin.qc" />
 		<property name="dao" value="com.raytheon.uf.edex.plugin.qc.dao.QCDao" />

nativeLib/rary.cots.jepp/jepp-2.3/.classpath

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="lib" path="ext/bsf.jar"/>
+	<classpathentry kind="lib" path="jep.jar"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+	<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+	<classpathentry kind="output" path="bin"/>
+</classpath>

nativeLib/rary.cots.jepp/jepp-2.3/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>jep</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+	</natures>
+</projectDescription>

nativeLib/rary.cots.jepp/jepp-2.3/Makefile

@@ -33,8 +33,8 @@ POST_INSTALL = :
 NORMAL_UNINSTALL = :
 PRE_UNINSTALL = :
 POST_UNINSTALL = :
-build_triplet = i686-pc-linux-gnu
+build_triplet = x86_64-unknown-linux-gnu
-host_triplet = i686-pc-linux-gnu
+host_triplet = x86_64-unknown-linux-gnu
 DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \
 	$(srcdir)/Makefile.in $(srcdir)/config.h.in \
 	$(srcdir)/jep.lsm.in $(srcdir)/jep.spec.in \
@@ -78,21 +78,21 @@ DIST_ARCHIVES = $(distdir).tar.gz
 GZIP_ENV = --best
 distuninstallcheck_listfiles = find . -type f -print
 distcleancheck_listfiles = find . -type f -print
-ACLOCAL = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run aclocal-1.9
+ACLOCAL = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run aclocal-1.9
 ALLOCA = 
 AMDEP_FALSE = #
 AMDEP_TRUE = 
-AMTAR = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run tar
+AMTAR = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run tar
 AR = ar
-AUTOCONF = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run autoconf
+AUTOCONF = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run autoconf
-AUTOHEADER = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run autoheader
+AUTOHEADER = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run autoheader
-AUTOMAKE = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run automake-1.9
+AUTOMAKE = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run automake-1.9
 AWK = gawk
 CC = gcc
 CCDEPMODE = depmode=gcc3
 CFLAGS = -g -O2
 CPP = gcc -E
-CPPFLAGS =  -I/common/njensen/awips/java/include -I/common/njensen/awips/java/include/linux -I/common/njensen/awips/python/include/python2.6 -I/common/njensen/awips/python/lib/python2.6/site-packages/numpy/core/include
+CPPFLAGS =  -I/awips2/java/include -I/awips2/java/include/linux -Wno-long-double -I/awips2/python/include/python2.7
 CXX = g++
 CXXCPP = g++ -E
 CXXDEPMODE = depmode=gcc3
@@ -112,19 +112,19 @@ INSTALL_DATA = ${INSTALL} -m 644
 INSTALL_PROGRAM = ${INSTALL}
 INSTALL_SCRIPT = ${INSTALL}
 INSTALL_STRIP_PROGRAM = ${SHELL} $(install_sh) -c -s
-ISODATE = 2010-01-21
+ISODATE = 2015-03-09
-JAVAC = /common/njensen/awips/java/bin/javac
+JAVAC = /awips2/java/bin/javac
-JAVAH = /common/njensen/awips/java/bin/javah
+JAVAH = /awips2/java/bin/javah
 JAVAX_SCRIPT_CLASSES = src/jep/JepScriptEngine.class src/jep/JepScriptEngineFactory.class
 JAVA_LDFLAGS = 
 JAVA_LIBS = 
 LDFLAGS = 
 LIBOBJS = 
-LIBS =  -L/common/njensen/awips/python/lib/python2.6/config -lpthread -ldl -lutil -lm -lpython2.6 
+LIBS =  -L/awips2/python/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7 
 LIBTOOL = $(SHELL) $(top_builddir)/libtool
 LN_S = ln -s
 LTLIBOBJS = 
-MAKEINFO = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run makeinfo
+MAKEINFO = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run makeinfo
 OBJEXT = o
 PACKAGE = jep
 PACKAGE_BUGREPORT = 
@@ -133,7 +133,7 @@ PACKAGE_STRING =
 PACKAGE_TARNAME = 
 PACKAGE_VERSION = 
 PATH_SEPARATOR = :
-PYTHON = /common/njensen/awips/python/bin/python
+PYTHON = /awips2/python/bin/python
 RANLIB = ranlib
 SET_MAKE = 
 SHELL = /bin/sh
@@ -155,21 +155,21 @@ am__quote =
 am__tar = ${AMTAR} chof - "$$tardir"
 am__untar = ${AMTAR} xf -
 bindir = ${exec_prefix}/bin
-build = i686-pc-linux-gnu
+build = x86_64-unknown-linux-gnu
 build_alias = 
-build_cpu = i686
+build_cpu = x86_64
 build_os = linux-gnu
-build_vendor = pc
+build_vendor = unknown
 datadir = ${prefix}/share
 exec_prefix = ${prefix}
-host = i686-pc-linux-gnu
+host = x86_64-unknown-linux-gnu
 host_alias = 
-host_cpu = i686
+host_cpu = x86_64
 host_os = linux-gnu
-host_vendor = pc
+host_vendor = unknown
 includedir = ${prefix}/include
 infodir = ${prefix}/info
-install_sh = /home/njensen/workspace/rary.cots.jepp/jepp-2.3/install-sh
+install_sh = /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/install-sh
 libdir = ${exec_prefix}/lib
 libexecdir = ${exec_prefix}/libexec
 localstatedir = ${prefix}/var
@@ -178,14 +178,14 @@ mkdir_p = mkdir -p --
 oldincludedir = /usr/include
 prefix = /usr/local
 program_transform_name = s,x,x,
-python_configdir = /common/njensen/awips/python/lib/python2.6/config
+python_configdir = /awips2/python/lib/python2.7/config
-python_execprefix = /common/njensen/awips/python
+python_execprefix = /awips2/python
-python_includespec = -I/common/njensen/awips/python/include/python2.6
+python_includespec = -I/awips2/python/include/python2.7
-python_libspec = -L/common/njensen/awips/python/lib/python2.6/config -lpthread -ldl -lutil -lm -lpython2.6 
+python_libspec = -L/awips2/python/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7 
-python_moduledir = /common/njensen/awips/python/lib/python2.6/site-packages
+python_moduledir = /awips2/python/lib/python2.7/site-packages
-python_moduleexecdir = /common/njensen/awips/python/lib/python2.6/site-packages
+python_moduleexecdir = /awips2/python/lib/python2.7/site-packages
-python_prefix = /common/njensen/awips/python
+python_prefix = /awips2/python
-python_version = 2.6
+python_version = 2.7
 sbindir = ${exec_prefix}/sbin
 sharedstatedir = ${prefix}/com
 sysconfdir = ${prefix}/etc
@@ -202,6 +202,7 @@ EXTRA_DIST = python.m4 \
 	src/jep/pyjclass.h \
 	src/jep/pyjfield.h \
 	src/jep/Jep.java \
+        src/jep/INumpyable.java \
 	src/jep/Run.java \
 	src/jep/BSFJepEngine.java \
 	src/jep/JepException.java \
@@ -796,6 +797,9 @@ src/jep/JepScriptEngineFactory.class: src/jep/JepScriptEngineFactory.java
 src/jep/InvocationHandler.class: src/jep/InvocationHandler.java
 	$(JAVAC) ${JAVACOPT} src/jep/InvocationHandler.java
 
+src/jep/INumpyable.class: src/jep/INumpyable.java
+	$(JAVAC) ${JAVACOPT} src/jep/INumpyable.java
+
 src/jep/invocationhandler.h: src/jep/InvocationHandler.class
 	$(JAVAH) -o src/jep/invocationhandler.h -classpath src/ jep.InvocationHandler
 # make sure the timestamp gets updated because javac might not
@@ -812,6 +816,7 @@ jep.jar:  \
 	src/jep/python/PyModule.class \
 	src/jep/python/PyClass.class \
 	src/jep/Jep.class \
+	src/jep/INumpyable.class \
 	src/jep/JepException.class \
 	src/jep/Test.class \
 	src/jep/Run.class \

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/INumpyable.java

@@ -1,6 +1,6 @@
 /*****************************************************************************************
  * COPYRIGHT (c), 2006-2008, RAYTHEON COMPANY
- * ALL RIGHTS RESERVED, An Unpublished Work 
+ * ALL RIGHTS RESERVED, An Unpublished Work
  *
  * RAYTHEON PROPRIETARY
  * If the end user is not the U.S. Government or any agency thereof, use
@@ -21,7 +21,12 @@ package jep;
 
 /**
  * Interface representing a Java object that can be transformed into a numpy
- * array.  
+ * array.
+ * 
+ * Only supports one and two dimensional arrays at this time.  Use python code
+ * to make arrays of more dimensions as necessary.
+ * 
+ * TODO rename to Numpyable
  * 
  * <pre>
  * SOFTWARE HISTORY
@@ -43,7 +48,7 @@ public interface INumpyable {
      * e.g. {float[], float[]}.  The result in python will then be a python
      * list, e.g. [numpy.ndarray(dtype=float32), numpy.ndarray(dtype=float32)].
      * 
-     * @return
+     * @return an array of arrays
      */
     public Object[] getNumpy();
 

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Jep.java

@@ -2,6 +2,7 @@ package jep;
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.List;
 
 import jep.python.PyModule;
 import jep.python.PyObject;
@@ -43,13 +44,26 @@ import jep.python.PyObject;
  */
 
 /*
-  August 2, 2012
+  Modified by Raytheon (c) 2008-2015 Raytheon Company. All Rights Reserved.
-  Modified by Raytheon (c) 2012 Raytheon Company. All Rights Reserved.
    Modifications marked and described by 'njensen'
 */
 
 public final class Jep {
     
+    private static final String THREAD_WARN_START = "JEP WARNING: "
+            + "Unsafe reuse of thread ";
+
+    private static final String THREAD_WARN_END = " for another Python interpreter.\n"
+            + " Potential issues can occur if you reuse the thread that JEP was"
+            + " initialized on or have multiple Jep instances on the same thread.";
+
+    /**
+     * Tracks which thread the Jep library was initialized on. That thread
+     * initialized the main python interpreter that all other subinterpreters
+     * will be created from.
+     */
+    private static long initializerThread = -1L;
+      
     private boolean closed = false;
     private long    tstate = 0;
     // all calls must originate from same thread
@@ -59,7 +73,7 @@ public final class Jep {
     private ClassLoader classLoader = null;
     
     // eval() storage.
-    private StringBuffer evalLines   = null;
+    private StringBuilder evalLines   = null;
     private boolean      interactive = false;
     
     // windows requires this as unix newline...
@@ -70,7 +84,34 @@ public final class Jep {
      * crashes in userland if jep is closed.
      *
      */
-    private ArrayList<PyObject> pythonObjects = new ArrayList<PyObject>();
+    private final List<PyObject> pythonObjects = new ArrayList<PyObject>();
+    
+    /**
+     * Tracks if this thread has been used for an interpreter before.  Using
+     * different interpreter instances on the same thread is iffy at best.  If
+     * you make use of CPython extensions (e.g. numpy) that use the GIL, then
+     * this gets even more risky and can potentially deadlock.
+     */
+    private final static ThreadLocal<Boolean> threadUsed = new ThreadLocal<Boolean>(){
+        @Override
+        protected Boolean initialValue(){
+            return false;
+        }
+    };
+    
+    // added by njensen
+    /**
+     * Loads the jep library (e.g. libjep.so or jep.dll) and initializes the
+     * main python interpreter that all subinterpreters will be created from.
+     */
+    public static synchronized Class<Jep> pyInitialize() {
+        if (initializerThread < 0) {
+            System.loadLibrary("jep");
+            initializerThread = Thread.currentThread().getId();
+            threadUsed.set(true);
+        }
+        return Jep.class;
+    }
     
     
     // -------------------------------------------------- constructors
@@ -119,6 +160,19 @@ public final class Jep {
     public Jep(boolean interactive,
                String includePath,
                ClassLoader cl) throws JepException {
+        // added by njensen
+        if (initializerThread < 0) {
+            throw new JepException("Jep Library must be initialized first"
+                    + ", please call pyInitialize()");
+        }
+        if (threadUsed.get()) {
+            /*
+             * TODO: Consider throwing an exception here to not allow this
+             * scenario through as it can result in very-hard-to-diagnose bugs.
+             */
+            System.err.println(THREAD_WARN_START
+                    + Thread.currentThread().getName() + THREAD_WARN_END);
+        } // end of njensen adds
         
         if(cl == null)
             this.classLoader = this.getClass().getClassLoader();
@@ -127,6 +181,7 @@ public final class Jep {
         
         this.interactive = interactive;
         this.tstate = init(this.classLoader);
+        threadUsed.set(true);  // added by njensen
         this.thread = Thread.currentThread();
         
         // why write C code if you don't have to? :-)
@@ -143,13 +198,6 @@ public final class Jep {
         }
     }
     
-    
-    // load shared library
-    static {
-        System.loadLibrary("jep");
-    }
-    
-    
     private native long init(ClassLoader classloader) throws JepException;
     
     
@@ -294,7 +342,7 @@ public final class Jep {
                 
                 // doesn't compile on it's own, append to eval
                 if(this.evalLines == null)
-                    this.evalLines = new StringBuffer();
+                    this.evalLines = new StringBuilder();
                 else
                     evalLines.append(LINE_SEP);
                 evalLines.append(str);
@@ -349,7 +397,7 @@ public final class Jep {
      * <pre>
      * Retrieves a python string object as a java array.
      *
-     * @param str a <code>String</code> 
+     * @param str a <code>String</code>
      * @return an <code>Object</code> array
      * @exception JepException if an error occurs
      */
@@ -370,7 +418,7 @@ public final class Jep {
      * <pre>
      * Retrieves a python string object as a java array.
      *
-     * @param str a <code>String</code> 
+     * @param str a <code>String</code>
      * @return an <code>Object</code> array
      * @exception JepException if an error occurs
      */
@@ -498,9 +546,12 @@ public final class Jep {
             throw new JepException("Jep has been closed.");
         isValidThread();
 
-	// njensen added all the instanceofs except Class
+        /*
-	if (v instanceof Class) {
+         * njensen added all the instanceofs except Class and the else to ensure
-            set(tstate, name, (Class) v);
+         * the correct python type in the interpreter
+         */
+        if (v instanceof Class) {
+            set(tstate, name, (Class<?>) v);
         } else if (v instanceof String) {
             set(name, ((String) v));
         } else if (v instanceof Float) {
@@ -520,14 +571,12 @@ public final class Jep {
         } else {
             set(tstate, name, v);
         }
-    
-    
     }
 
     private native void set(long tstate, String name, Object v)
         throws JepException;
 
-    private native void set(long tstate, String name, Class v)
+    private native void set(long tstate, String name, Class<?> v)
         throws JepException;
 
     /**
@@ -593,7 +642,7 @@ public final class Jep {
             throw new JepException("Jep has been closed.");
         isValidThread();
         
-        set(tstate, name, (int) v);
+        set(tstate, name, v);
     }
     
     private native void set(long tstate, String name, int v)
@@ -644,7 +693,7 @@ public final class Jep {
             throw new JepException("Jep has been closed.");
         isValidThread();
         
-        set(tstate, name, (int) b);
+        set(tstate, name, b);
     }
 
     
@@ -820,39 +869,48 @@ public final class Jep {
     private native void set(long tstate, String name, double[] v)
         throws JepException;
 
-    // added by njensen
+    /*
-    public void setNumpy(String name, float[] v, int nx, int ny) throws JepException {
+     * added by njensen.
-	if(this.closed)
+     * TODO: rename all methods to setNumpy
-	    throw new JepException("Jep has been closed.");
+     * TODO: Add javadoc
-	isValidThread();
+     * TODO: add support for short[], long[], and double[]
+     */
+    public void setNumpy(String name, float[] v, int nx, int ny)
+            throws JepException {
+        if (this.closed)
+            throw new JepException("Jep has been closed.");
+        isValidThread();
 
-	setNumeric(tstate, name, v, nx, ny);
+        setNumeric(tstate, name, v, nx, ny);
     }
 
-    private native void setNumeric(long tstate, String name, float[] v, int nx, int ny)
+    private native void setNumeric(long tstate, String name, float[] v, int nx,
-	throws JepException;
+            int ny) throws JepException;
+
+    public void setNumpy(String name, int[] v, int nx, int ny)
+            throws JepException {
+        if (this.closed)
+            throw new JepException("Jep has been closed.");
+        isValidThread();
+
+        setNumeric(tstate, name, v, nx, ny);
+    }
+
+    private native void setNumeric(long tstate, String name, int[] v, int nx,
+            int ny) throws JepException;
+
+    public void setNumpy(String name, byte[] v, int nx, int ny)
+            throws JepException {
+        if (this.closed)
+            throw new JepException("Jep has been closed.");
+        isValidThread();
+
+        setNumeric(tstate, name, v, nx, ny);
+    }
+
+    private native void setNumeric(long tstate, String name, byte[] v, int nx,
+            int ny) throws JepException;
     // end of added by njensen
-    public void setNumpy(String name, int[] v, int nx, int ny) throws JepException {
-        if(this.closed)
-            throw new JepException("Jep has been closed.");
-        isValidThread();
-
-        setNumeric(tstate, name, v, nx, ny);
-    }
-
-    private native void setNumeric(long tstate, String name, int[] v, int nx, int ny)
-        throws JepException;
-    
-    public void setNumpy(String name, byte[] v, int nx, int ny) throws JepException {
-        if(this.closed)
-            throw new JepException("Jep has been closed.");
-        isValidThread();
-
-        setNumeric(tstate, name, v, nx, ny);
-    }
-    
-    private native void setNumeric(long tstate, String name, byte[] v, int nx, int ny)
-    throws JepException;
 
 
 
@@ -892,6 +950,11 @@ public final class Jep {
         this.closed = true;
         this.close(tstate);
         this.tstate = 0;
+        /*
+         * added by njensen, untested on how safe this really is to allow
+         * reuse after the previous instance was closed
+         */
+        threadUsed.set(false);
     }
 
     private native void close(long tstate);
@@ -901,6 +964,7 @@ public final class Jep {
      * Describe <code>finalize</code> method here.
      *
      */
+    @Override
     protected void finalize() {
         this.close();
     }