Omaha #4234 initialize the main python interpreter on a different thread

than the sub-interpreters, or warn when that happens

Change-Id: Id4d2c2401c96f6d884e5c468419f2d5893f6eafd

Former-commit-id: 5386e7fabcfbac366bb5335f601bcd0719b4f32c
This commit is contained in:
Nate Jensen 2015-03-10 15:41:32 -05:00
parent cbb92fa497
commit 7318d0e603
10 changed files with 191 additions and 87 deletions

View file

@ -139,7 +139,7 @@ wrapper.java.additional.log.5=-Djava.util.logging.config.file=${EDEX_HOME}/conf/
# errors reported by thrift sometimes when the stream is corrupt/incorrect
wrapper.java.additional.thrift.maxStreamSize=-Dthrift.stream.maxsize=200
wrapper.java.additional.retain.failed=-Dretain.failed.data=${RETAIN_FAILED}
#wrapper.java.additional.retain.failed=-Dretain.failed.data=${RETAIN_FAILED}
# enables yourkit profiling, determined by flag to start.sh
wrapper.java.additional.profile.1=${PROFILER_PARAM_1}

View file

@ -8,7 +8,8 @@
<constructor-arg ref="fetchATSrv" />
</bean>
<bean id="gfeSitesActiveRequest" factory-bean="siteAwareRegistry" factory-method="register">
<bean id="gfeSitesActiveRequest" factory-bean="siteAwareRegistry" factory-method="register"
depends-on="jepInit">
<constructor-arg ref="gfeSiteActivation"/>
</bean>

View file

@ -12,7 +12,8 @@
<bean id="smartInitQueue" class="com.raytheon.edex.plugin.gfe.smartinit.SmartInitQueue" factory-method="createQueue"/>
<bean id="gfeSitesActiveIngest" factory-bean="siteAwareRegistry" factory-method="register" depends-on="smartInitQueue">
<bean id="gfeSitesActiveIngest" factory-bean="siteAwareRegistry" factory-method="register"
depends-on="jepInit, smartInitQueue">
<constructor-arg ref="gfeSiteActivation"/>
</bean>

View file

@ -31,6 +31,7 @@
<mode name="ingestGrib">
<include>time-common.xml</include>
<include>auth-common.xml</include>
<include>python-common.xml</include>
<include>grib-decode.xml</include>
<include>ncgrib-file-endpoint.xml</include>
<include>grid-staticdata-process.xml</include>

View file

@ -3,7 +3,8 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
<bean id="qcProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties">
<bean id="qcProperties" class="com.raytheon.uf.common.dataplugin.PluginProperties"
depends-on="jepInit">
<property name="pluginName" value="qc" />
<property name="pluginFQN" value="com.raytheon.uf.common.dataplugin.qc" />
<property name="dao" value="com.raytheon.uf.edex.plugin.qc.dao.QCDao" />

View file

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src"/>
<classpathentry kind="lib" path="ext/bsf.jar"/>
<classpathentry kind="lib" path="jep.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
<classpathentry kind="output" path="bin"/>
</classpath>

View file

@ -0,0 +1,17 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>jep</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

View file

@ -33,8 +33,8 @@ POST_INSTALL = :
NORMAL_UNINSTALL = :
PRE_UNINSTALL = :
POST_UNINSTALL = :
build_triplet = i686-pc-linux-gnu
host_triplet = i686-pc-linux-gnu
build_triplet = x86_64-unknown-linux-gnu
host_triplet = x86_64-unknown-linux-gnu
DIST_COMMON = README $(am__configure_deps) $(srcdir)/Makefile.am \
$(srcdir)/Makefile.in $(srcdir)/config.h.in \
$(srcdir)/jep.lsm.in $(srcdir)/jep.spec.in \
@ -78,21 +78,21 @@ DIST_ARCHIVES = $(distdir).tar.gz
GZIP_ENV = --best
distuninstallcheck_listfiles = find . -type f -print
distcleancheck_listfiles = find . -type f -print
ACLOCAL = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run aclocal-1.9
ACLOCAL = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run aclocal-1.9
ALLOCA =
AMDEP_FALSE = #
AMDEP_TRUE =
AMTAR = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run tar
AMTAR = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run tar
AR = ar
AUTOCONF = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run autoconf
AUTOHEADER = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run autoheader
AUTOMAKE = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run automake-1.9
AUTOCONF = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run autoconf
AUTOHEADER = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run autoheader
AUTOMAKE = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run automake-1.9
AWK = gawk
CC = gcc
CCDEPMODE = depmode=gcc3
CFLAGS = -g -O2
CPP = gcc -E
CPPFLAGS = -I/common/njensen/awips/java/include -I/common/njensen/awips/java/include/linux -I/common/njensen/awips/python/include/python2.6 -I/common/njensen/awips/python/lib/python2.6/site-packages/numpy/core/include
CPPFLAGS = -I/awips2/java/include -I/awips2/java/include/linux -Wno-long-double -I/awips2/python/include/python2.7
CXX = g++
CXXCPP = g++ -E
CXXDEPMODE = depmode=gcc3
@ -112,19 +112,19 @@ INSTALL_DATA = ${INSTALL} -m 644
INSTALL_PROGRAM = ${INSTALL}
INSTALL_SCRIPT = ${INSTALL}
INSTALL_STRIP_PROGRAM = ${SHELL} $(install_sh) -c -s
ISODATE = 2010-01-21
JAVAC = /common/njensen/awips/java/bin/javac
JAVAH = /common/njensen/awips/java/bin/javah
ISODATE = 2015-03-09
JAVAC = /awips2/java/bin/javac
JAVAH = /awips2/java/bin/javah
JAVAX_SCRIPT_CLASSES = src/jep/JepScriptEngine.class src/jep/JepScriptEngineFactory.class
JAVA_LDFLAGS =
JAVA_LIBS =
LDFLAGS =
LIBOBJS =
LIBS = -L/common/njensen/awips/python/lib/python2.6/config -lpthread -ldl -lutil -lm -lpython2.6
LIBS = -L/awips2/python/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7
LIBTOOL = $(SHELL) $(top_builddir)/libtool
LN_S = ln -s
LTLIBOBJS =
MAKEINFO = ${SHELL} /home/njensen/workspace/rary.cots.jepp/jepp-2.3/missing --run makeinfo
MAKEINFO = ${SHELL} /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/missing --run makeinfo
OBJEXT = o
PACKAGE = jep
PACKAGE_BUGREPORT =
@ -133,7 +133,7 @@ PACKAGE_STRING =
PACKAGE_TARNAME =
PACKAGE_VERSION =
PATH_SEPARATOR = :
PYTHON = /common/njensen/awips/python/bin/python
PYTHON = /awips2/python/bin/python
RANLIB = ranlib
SET_MAKE =
SHELL = /bin/sh
@ -155,21 +155,21 @@ am__quote =
am__tar = ${AMTAR} chof - "$$tardir"
am__untar = ${AMTAR} xf -
bindir = ${exec_prefix}/bin
build = i686-pc-linux-gnu
build = x86_64-unknown-linux-gnu
build_alias =
build_cpu = i686
build_cpu = x86_64
build_os = linux-gnu
build_vendor = pc
build_vendor = unknown
datadir = ${prefix}/share
exec_prefix = ${prefix}
host = i686-pc-linux-gnu
host = x86_64-unknown-linux-gnu
host_alias =
host_cpu = i686
host_cpu = x86_64
host_os = linux-gnu
host_vendor = pc
host_vendor = unknown
includedir = ${prefix}/include
infodir = ${prefix}/info
install_sh = /home/njensen/workspace/rary.cots.jepp/jepp-2.3/install-sh
install_sh = /common/njensen/git/15.1/AWIPS2_baseline/nativeLib/rary.cots.jepp/jepp-2.3/install-sh
libdir = ${exec_prefix}/lib
libexecdir = ${exec_prefix}/libexec
localstatedir = ${prefix}/var
@ -178,14 +178,14 @@ mkdir_p = mkdir -p --
oldincludedir = /usr/include
prefix = /usr/local
program_transform_name = s,x,x,
python_configdir = /common/njensen/awips/python/lib/python2.6/config
python_execprefix = /common/njensen/awips/python
python_includespec = -I/common/njensen/awips/python/include/python2.6
python_libspec = -L/common/njensen/awips/python/lib/python2.6/config -lpthread -ldl -lutil -lm -lpython2.6
python_moduledir = /common/njensen/awips/python/lib/python2.6/site-packages
python_moduleexecdir = /common/njensen/awips/python/lib/python2.6/site-packages
python_prefix = /common/njensen/awips/python
python_version = 2.6
python_configdir = /awips2/python/lib/python2.7/config
python_execprefix = /awips2/python
python_includespec = -I/awips2/python/include/python2.7
python_libspec = -L/awips2/python/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7
python_moduledir = /awips2/python/lib/python2.7/site-packages
python_moduleexecdir = /awips2/python/lib/python2.7/site-packages
python_prefix = /awips2/python
python_version = 2.7
sbindir = ${exec_prefix}/sbin
sharedstatedir = ${prefix}/com
sysconfdir = ${prefix}/etc
@ -202,6 +202,7 @@ EXTRA_DIST = python.m4 \
src/jep/pyjclass.h \
src/jep/pyjfield.h \
src/jep/Jep.java \
src/jep/INumpyable.java \
src/jep/Run.java \
src/jep/BSFJepEngine.java \
src/jep/JepException.java \
@ -796,6 +797,9 @@ src/jep/JepScriptEngineFactory.class: src/jep/JepScriptEngineFactory.java
src/jep/InvocationHandler.class: src/jep/InvocationHandler.java
$(JAVAC) ${JAVACOPT} src/jep/InvocationHandler.java
src/jep/INumpyable.class: src/jep/INumpyable.java
$(JAVAC) ${JAVACOPT} src/jep/INumpyable.java
src/jep/invocationhandler.h: src/jep/InvocationHandler.class
$(JAVAH) -o src/jep/invocationhandler.h -classpath src/ jep.InvocationHandler
# make sure the timestamp gets updated because javac might not
@ -812,6 +816,7 @@ jep.jar: \
src/jep/python/PyModule.class \
src/jep/python/PyClass.class \
src/jep/Jep.class \
src/jep/INumpyable.class \
src/jep/JepException.class \
src/jep/Test.class \
src/jep/Run.class \

View file

@ -23,6 +23,11 @@ package jep;
* Interface representing a Java object that can be transformed into a numpy
* array.
*
* Only supports one and two dimensional arrays at this time. Use python code
* to make arrays of more dimensions as necessary.
*
* TODO rename to Numpyable
*
* <pre>
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
@ -43,7 +48,7 @@ public interface INumpyable {
* e.g. {float[], float[]}. The result in python will then be a python
* list, e.g. [numpy.ndarray(dtype=float32), numpy.ndarray(dtype=float32)].
*
* @return
* @return an array of arrays
*/
public Object[] getNumpy();

View file

@ -2,6 +2,7 @@ package jep;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import jep.python.PyModule;
import jep.python.PyObject;
@ -43,13 +44,26 @@ import jep.python.PyObject;
*/
/*
August 2, 2012
Modified by Raytheon (c) 2012 Raytheon Company. All Rights Reserved.
Modified by Raytheon (c) 2008-2015 Raytheon Company. All Rights Reserved.
Modifications marked and described by 'njensen'
*/
public final class Jep {
private static final String THREAD_WARN_START = "JEP WARNING: "
+ "Unsafe reuse of thread ";
private static final String THREAD_WARN_END = " for another Python interpreter.\n"
+ " Potential issues can occur if you reuse the thread that JEP was"
+ " initialized on or have multiple Jep instances on the same thread.";
/**
* Tracks which thread the Jep library was initialized on. That thread
* initialized the main python interpreter that all other subinterpreters
* will be created from.
*/
private static long initializerThread = -1L;
private boolean closed = false;
private long tstate = 0;
// all calls must originate from same thread
@ -59,7 +73,7 @@ public final class Jep {
private ClassLoader classLoader = null;
// eval() storage.
private StringBuffer evalLines = null;
private StringBuilder evalLines = null;
private boolean interactive = false;
// windows requires this as unix newline...
@ -70,7 +84,34 @@ public final class Jep {
* crashes in userland if jep is closed.
*
*/
private ArrayList<PyObject> pythonObjects = new ArrayList<PyObject>();
private final List<PyObject> pythonObjects = new ArrayList<PyObject>();
/**
* Tracks if this thread has been used for an interpreter before. Using
* different interpreter instances on the same thread is iffy at best. If
* you make use of CPython extensions (e.g. numpy) that use the GIL, then
* this gets even more risky and can potentially deadlock.
*/
private final static ThreadLocal<Boolean> threadUsed = new ThreadLocal<Boolean>(){
@Override
protected Boolean initialValue(){
return false;
}
};
// added by njensen
/**
* Loads the jep library (e.g. libjep.so or jep.dll) and initializes the
* main python interpreter that all subinterpreters will be created from.
*/
public static synchronized Class<Jep> pyInitialize() {
if (initializerThread < 0) {
System.loadLibrary("jep");
initializerThread = Thread.currentThread().getId();
threadUsed.set(true);
}
return Jep.class;
}
// -------------------------------------------------- constructors
@ -119,6 +160,19 @@ public final class Jep {
public Jep(boolean interactive,
String includePath,
ClassLoader cl) throws JepException {
// added by njensen
if (initializerThread < 0) {
throw new JepException("Jep Library must be initialized first"
+ ", please call pyInitialize()");
}
if (threadUsed.get()) {
/*
* TODO: Consider throwing an exception here to not allow this
* scenario through as it can result in very-hard-to-diagnose bugs.
*/
System.err.println(THREAD_WARN_START
+ Thread.currentThread().getName() + THREAD_WARN_END);
} // end of njensen adds
if(cl == null)
this.classLoader = this.getClass().getClassLoader();
@ -127,6 +181,7 @@ public final class Jep {
this.interactive = interactive;
this.tstate = init(this.classLoader);
threadUsed.set(true); // added by njensen
this.thread = Thread.currentThread();
// why write C code if you don't have to? :-)
@ -143,13 +198,6 @@ public final class Jep {
}
}
// load shared library
static {
System.loadLibrary("jep");
}
private native long init(ClassLoader classloader) throws JepException;
@ -294,7 +342,7 @@ public final class Jep {
// doesn't compile on it's own, append to eval
if(this.evalLines == null)
this.evalLines = new StringBuffer();
this.evalLines = new StringBuilder();
else
evalLines.append(LINE_SEP);
evalLines.append(str);
@ -498,9 +546,12 @@ public final class Jep {
throw new JepException("Jep has been closed.");
isValidThread();
// njensen added all the instanceofs except Class
/*
* njensen added all the instanceofs except Class and the else to ensure
* the correct python type in the interpreter
*/
if (v instanceof Class) {
set(tstate, name, (Class) v);
set(tstate, name, (Class<?>) v);
} else if (v instanceof String) {
set(name, ((String) v));
} else if (v instanceof Float) {
@ -520,14 +571,12 @@ public final class Jep {
} else {
set(tstate, name, v);
}
}
private native void set(long tstate, String name, Object v)
throws JepException;
private native void set(long tstate, String name, Class v)
private native void set(long tstate, String name, Class<?> v)
throws JepException;
/**
@ -593,7 +642,7 @@ public final class Jep {
throw new JepException("Jep has been closed.");
isValidThread();
set(tstate, name, (int) v);
set(tstate, name, v);
}
private native void set(long tstate, String name, int v)
@ -644,7 +693,7 @@ public final class Jep {
throw new JepException("Jep has been closed.");
isValidThread();
set(tstate, name, (int) b);
set(tstate, name, b);
}
@ -820,8 +869,14 @@ public final class Jep {
private native void set(long tstate, String name, double[] v)
throws JepException;
// added by njensen
public void setNumpy(String name, float[] v, int nx, int ny) throws JepException {
/*
* added by njensen.
* TODO: rename all methods to setNumpy
* TODO: Add javadoc
* TODO: add support for short[], long[], and double[]
*/
public void setNumpy(String name, float[] v, int nx, int ny)
throws JepException {
if (this.closed)
throw new JepException("Jep has been closed.");
isValidThread();
@ -829,30 +884,33 @@ public final class Jep {
setNumeric(tstate, name, v, nx, ny);
}
private native void setNumeric(long tstate, String name, float[] v, int nx, int ny)
throws JepException;
private native void setNumeric(long tstate, String name, float[] v, int nx,
int ny) throws JepException;
public void setNumpy(String name, int[] v, int nx, int ny)
throws JepException {
if (this.closed)
throw new JepException("Jep has been closed.");
isValidThread();
setNumeric(tstate, name, v, nx, ny);
}
private native void setNumeric(long tstate, String name, int[] v, int nx,
int ny) throws JepException;
public void setNumpy(String name, byte[] v, int nx, int ny)
throws JepException {
if (this.closed)
throw new JepException("Jep has been closed.");
isValidThread();
setNumeric(tstate, name, v, nx, ny);
}
private native void setNumeric(long tstate, String name, byte[] v, int nx,
int ny) throws JepException;
// end of added by njensen
public void setNumpy(String name, int[] v, int nx, int ny) throws JepException {
if(this.closed)
throw new JepException("Jep has been closed.");
isValidThread();
setNumeric(tstate, name, v, nx, ny);
}
private native void setNumeric(long tstate, String name, int[] v, int nx, int ny)
throws JepException;
public void setNumpy(String name, byte[] v, int nx, int ny) throws JepException {
if(this.closed)
throw new JepException("Jep has been closed.");
isValidThread();
setNumeric(tstate, name, v, nx, ny);
}
private native void setNumeric(long tstate, String name, byte[] v, int nx, int ny)
throws JepException;
@ -892,6 +950,11 @@ public final class Jep {
this.closed = true;
this.close(tstate);
this.tstate = 0;
/*
* added by njensen, untested on how safe this really is to allow
* reuse after the previous instance was closed
*/
threadUsed.set(false);
}
private native void close(long tstate);
@ -901,6 +964,7 @@ public final class Jep {
* Describe <code>finalize</code> method here.
*
*/
@Override
protected void finalize() {
this.close();
}