xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Omaha #4259 fix jep unit test threading unsafety

Browse source 
Change-Id: I13f58d8dcf3e3d074c3586d2682ad3fa9809a3fe

Former-commit-id: 24922b47c679227489740da2979d10d9f81088bf
This commit is contained in:
Nate Jensen 2015-04-15 17:47:57 -05:00
parent c54c7e6173
commit eabd52c620
5 changed files with 212 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Jep.java
View file

@ -1,5 +1,6 @@
package jep;
import java.io.Closeable;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
@ -42,7 +43,7 @@ import jep.python.PyObject;
* @author [mrjohnson0 at sourceforge.net] Mike Johnson
* @version $Id$
*/
public final class Jep {
public final class Jep implements Closeable {
private static final String THREAD_WARN = "JEP WARNING: "
+ "Unsafe reuse of thread ";
@ -204,13 +205,16 @@ public final class Jep {
ClassEnquirer ce) throws JepException {
if (threadUsed.get()) {
/*
* TODO: Consider throwing an exception for this situation instead
* of printing a warning, as it can result in very-hard-to-diagnose
* bugs such as GIL-related freezes or misleading error messages.
* TODO: Throw a JepException if this is detected. This is
* inherently unsafe, the thread state information inside
* Python can get screwed up if there's more than one started/open
* Jep on the same thread at any given time. This remains a
* warning for the time being to provide time for applications
* to be updated to avoid this scenario.
*/
Thread current = Thread.currentThread();
String warn = THREAD_WARN + current.getName() + REUSE_WARN;
System.err.println(warn);
System.err.println(warn);
}
if(cl == null)
@ -934,6 +938,7 @@ public final class Jep {
* Shutdown python interpreter. Make sure you call this.
*
*/
@Override
public synchronized void close() {
if(this.closed)
return;

28
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Run.java
View file

@ -1,5 +1,7 @@
package jep;
import java.io.File;
/**
* <pre>
* Run.java - Execute a Python script.
@ -50,20 +52,28 @@ public class Run {
public static int run(boolean eventDispatch) {
Jep jep = null;
try {
jep = new Jep(false, ".");
// "set" by eval'ing it
jep.eval("import sys; sys.argv = argv = " + scriptArgv);
jep.runScript(file);
}
catch(Throwable t) {
// "set" by eval'ing it
jep.eval("import sys; sys.argv = argv = " + scriptArgv);
if (!file.endsWith("jep" + File.separator + "console.py")) {
jep.runScript(file);
} else {
// don't use console's __main__ so we can reuse the interpreter
jep.setInteractive(true);
jep.set("jepInstance", jep);
jep.eval("from jep import console");
jep.eval("console.prompt(jepInstance)");
}
} catch (Throwable t) {
t.printStackTrace();
if(jep != null)
if (jep != null)
jep.close();
return 1;
}
}
try {
if(interactive) {

40
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Test.java
View file

@ -16,6 +16,7 @@ import jep.python.*;
* @version $Id$
*/
public class Test implements Runnable {
private Jep jep = null;
private boolean testEval = false;
@ -452,6 +453,45 @@ public class Test implements Runnable {
return Thread.currentThread().getClass();
}
public static void testRestrictedClassLoader() throws Throwable {
final Throwable[] t = new Throwable[1];
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
Jep jep = null;
try {
jep = new Jep(true, "", restrictedClassLoader);
jep.eval("from java.io import File");
} catch (Throwable th) {
t[0] = th;
} finally {
if(jep != null) {
jep.close();
}
synchronized (Test.class) {
Test.class.notify();
}
}
}
});
synchronized (Test.class) {
thread.start();
try {
Test.class.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
if(t[0] == null) {
throw new RuntimeException("Did not throw classloader exception!");
} else if(!t[0].getMessage().contains("ImportError")) {
throw t[0];
}
}
public static void main(String argv[]) throws Throwable {
Jep jep = new Jep();
try {

181
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/TestNumpy.java
View file

@ -3,9 +3,8 @@ package jep;
import java.io.File;
/**
* TestNumpy.java. Runs a variety of simple tests to verify numpy interactions
* are working correctly. run() is called by setup.py test, or you can call the
* java main() method directly for other tests.
* TestNumpy.java. Runs a variety of simple tests to verify numpy interactions
* are working correctly.
*
*
* Created: Wed Apr 08 2015
@ -13,38 +12,65 @@ import java.io.File;
* @author [ndjensen at gmail.com] Nate Jensen
* @version $Id$
*/
public class TestNumpy implements Runnable {
protected Jep jep = null;
public class TestNumpy {
// set to a high number to test for memory leaks
private static final int REPEAT = 1; // 0000000;
private static boolean PRINT = false;
private static boolean PRINT = true;
/**
* Calls testSetAndGet(Jep) on a separate thread to avoid threading
* issues. Waits for those results before returning.
* @throws Throwable
*/
public void testSetAndGet() throws Throwable
{
final Throwable[] t = new Throwable[1];
Thread thread = new Thread(new Runnable() {
@Override
public void run() {
try {
File pwd = new File(".");
/*
* Anytime you start a new Jep interpreter and import numpy within
* it, even if you call close() on the interpreter you will leak
* some native memory. Therefore, for now, do NOT new up the
* interpreter and close it within the for loop.
*/
jep = new Jep(false, pwd.getAbsolutePath());
for (int i = 0; i < REPEAT; i++) {
testSetAndGet();
@Override
public void run() {
Jep jep = null;
try {
jep = new Jep(true);
} catch (Throwable th) {
t[0] = th;
} finally {
if(jep != null) {
jep.close();
}
synchronized (TestNumpy.this) {
TestNumpy.this.notify();
}
}
}
});
synchronized (TestNumpy.this) {
thread.start();
try {
TestNumpy.this.wait();
} catch (InterruptedException e) {
e.printStackTrace();
}
} catch (JepException e) {
e.printStackTrace();
} finally {
if (jep != null)
jep.close();
}
if(t[0] != null) {
throw t[0];
}
}
public void testSetAndGet() throws JepException {
/**
* Sets NDArrays in a Jep interpreter, then gets them and verifies
* the conversion in both directions is safe, ie produces a symmetrical
* object despite a different reference/instance.
* @param jep
* @throws JepException
*/
public void testSetAndGet(Jep jep) throws JepException {
int[] dimensions = new int[] { 4 };
// test boolean[]
@ -178,6 +204,12 @@ public class TestNumpy implements Runnable {
}
}
/**
* Called from python to verify that a Java method's return type of NDArray
* can be auto-converted to a numpy ndarray.
* @param array
* @return a copy of the data + 5
*/
public NDArray<int[]> testArgAndReturn(NDArray<int[]> array) {
int[] data = array.getData();
int[] newData = new int[data.length];
@ -188,29 +220,32 @@ public class TestNumpy implements Runnable {
return new NDArray<int[]>(newData, array.getDimensions());
}
// should only be called from main(), not from python unittests
public void runPythonSide() {
/**
* Helper method to support running the main() method to run from Java
* instead of python
* @param jep
*/
public void runFromJava(Jep jep) {
try {
File pwd = new File("tests");
/*
* Anytime you start a new Jep interpreter and import numpy within
* it, even if you call close() on the interpreter you will leak
* some native memory. Therefore, for now, do NOT new up the
* interpreter and close it within the for loop.
*/
jep = new Jep(true, pwd.getAbsolutePath());
jep.eval("import test_numpy");
jep.eval("v = test_numpy.TestNumpy('testArgReturn')");
jep.eval("v.setUp()");
jep.eval("v = test_numpy.TestNumpy('testArgReturn')");
jep.eval("v.setUp()");
for (int i = 0; i < REPEAT; i++) {
this.testSetAndGet(jep);
}
for (int i = 0; i < REPEAT; i++) {
jep.eval("v.testArgReturn()");
}
System.out.println("return NDArray from Java checked out ok");
for (int i = 0; i < REPEAT; i++) {
jep.eval("v.testMultiDimensional()");
}
System.out.println("multi dimensional arrays checked out ok");
for (int i = 0; i < REPEAT; i++) {
jep.eval("v.testArrayParams()");
jep.eval("v.testArrayParams()");
}
System.out.println("Passing ndarrays to Java method as Java primitive[] checked out ok");
} catch (JepException e) {
e.printStackTrace();
} finally {
@ -219,34 +254,79 @@ public class TestNumpy implements Runnable {
}
}
/**
* Verifies a numpy.ndarray of bool can automatically convert to a method
* arg of boolean[]
* @param array
* @return true on success
*/
public boolean callBooleanMethod(boolean[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of byte can automatically convert to a method
* arg of byte[]
* @param array
* @return true on success
*/
public boolean callByteMethod(byte[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of int16 can automatically convert to a method
* arg of short[]
* @param array
* @return true on success
*/
public boolean callShortMethod(short[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of int32 can automatically convert to a method
* arg of int[]
* @param array
* @return true on success
*/
public boolean callIntMethod(int[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of int64 can automatically convert to a method
* arg of long[]
* @param array
* @return true on success
*/
public boolean callLongMethod(long[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of float32 can automatically convert to a method
* arg of float[]
* @param array
* @return true on success
*/
public boolean callFloatMethod(float[] array) {
return array != null;
}
/**
* Verifies a numpy.ndarray of float64 can automatically convert to a method
* arg of double[]
* @param array
* @return true on success
*/
public boolean callDoubleMethod(double[] array) {
return array != null;
}
/**
* Verifies that an NDArray will not allow bad/dangerous constructor args.
*/
public void testNDArraySafety() {
float[][] f = new float[15][];
int[] dims = new int[] { 15, 20 };
@ -257,7 +337,7 @@ public class TestNumpy implements Runnable {
"NDArray should have failed instantiation");
} catch (IllegalArgumentException e) {
if (PRINT) {
e.printStackTrace();
System.out.println("NDArray blocked bad type args");
}
}
@ -269,7 +349,7 @@ public class TestNumpy implements Runnable {
"NDArray should have failed instantiation");
} catch (IllegalArgumentException e) {
if (PRINT) {
e.printStackTrace();
System.out.println("NDArray blocked bad dimensions args");
}
}
}
@ -281,10 +361,21 @@ public class TestNumpy implements Runnable {
* @param args
*/
public static void main(String[] args) {
TestNumpy test = new TestNumpy();
test.run();
test.runPythonSide();
test.testNDArraySafety();
File pwd = new File("tests");
TestNumpy test = null;
Jep jep = null;
try {
test = new TestNumpy();
jep = new Jep(false, pwd.getPath());
test.runFromJava(jep);
test.testNDArraySafety();
} catch (JepException e) {
e.printStackTrace();
} finally {
if (jep != null) {
jep.close();
}
}
}
}

23
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/TestNumpyGILFreeze.java
View file

@ -4,10 +4,9 @@ package jep;
* A test class that illustrates how numpy's floating point error handling can
* cause python to deadlock trying to acquire the GIL. More specifically,
* ufunc_object.c calls NPY_ALLOW_C_API which attempts to acquire the GIL.
* Python is confused about the thread state and GIL state due to the top level
* interpreter being initialized on the same thread as the sub-interpreter(s).
* Python is confused about the thread state and GIL state due to more than
* one Jep interpreter on the same thread.
*
* Please see the TODO below for how to trigger the freeze.
*
* Created: Thu Apr 09 2015
*
@ -28,21 +27,12 @@ public class TestNumpyGILFreeze {
*
* @param args
*/
public static void main(String[] args) {
/*
* TODO: To show the freeze, you need to alter Jep.TopInterpreter to
* NOT use a separate thread for System.loadLibrary("jep");
*/
createJepAndUseNumpy();
createJepAndUseNumpy();
System.out.println("java main() finished");
}
public static void createJepAndUseNumpy() {
public static void main(String[] args) {
Jep jep = null;
try {
Jep jep0 = new Jep(true);
jep = new Jep(true);
jep0.close();
jep.eval("import numpy");
/*
* If error conditions are set to ignore, we will not reach the
@ -51,7 +41,7 @@ public class TestNumpyGILFreeze {
*/
jep.eval("numpy.seterr(under='print')");
jep.eval(UNDERFLOW);
jep.eval("forceUnderflow()");
jep.eval("forceUnderflow()"); // this line will freeze
System.out.println("returned from python interpreter");
} catch (JepException e) {
e.printStackTrace();
@ -61,6 +51,7 @@ public class TestNumpyGILFreeze {
jep.close();
}
}
System.out.println("java main() finished");
}
}