Omaha #4259 fix jep unit test threading unsafety

Change-Id: I13f58d8dcf3e3d074c3586d2682ad3fa9809a3fe

Former-commit-id: 24922b47c679227489740da2979d10d9f81088bf

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Jep.java

@@ -1,5 +1,6 @@
 package jep;
 
+import java.io.Closeable;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
@@ -42,7 +43,7 @@ import jep.python.PyObject;
  * @author [mrjohnson0 at sourceforge.net] Mike Johnson
  * @version $Id$
  */
-public final class Jep {
+public final class Jep implements Closeable {
     
     private static final String THREAD_WARN = "JEP WARNING: "
             + "Unsafe reuse of thread ";
@@ -204,9 +205,12 @@ public final class Jep {
                ClassEnquirer ce) throws JepException {                     
         if (threadUsed.get()) {
             /*
-             * TODO: Consider throwing an exception for this situation instead
+             * TODO: Throw a JepException if this is detected.  This is
-             * of printing a warning, as it can result in very-hard-to-diagnose
+             * inherently unsafe, the thread state information inside
-             * bugs such as GIL-related freezes or misleading error messages.
+             * Python can get screwed up if there's more than one started/open
+             * Jep on the same thread at any given time.  This remains a 
+             * warning for the time being to provide time for applications
+             * to be updated to avoid this scenario.             
              */
             Thread current = Thread.currentThread();
             String warn = THREAD_WARN + current.getName() + REUSE_WARN;
@@ -934,6 +938,7 @@ public final class Jep {
      * Shutdown python interpreter. Make sure you call this.
      *
      */
+    @Override
     public synchronized void close() {
         if(this.closed)
             return;

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Run.java

@@ -1,5 +1,7 @@
 package jep;
 
+import java.io.File;
+
 /**
  * <pre>
  * Run.java - Execute a Python script.
@@ -50,16 +52,24 @@ public class Run {
     public static int run(boolean eventDispatch) {        
         Jep jep = null;
         
+        
         try {
             jep = new Jep(false, ".");
 
             // "set" by eval'ing it                        
             jep.eval("import sys; sys.argv = argv = " + scriptArgv);            
+            if (!file.endsWith("jep" + File.separator + "console.py")) {
                 jep.runScript(file);
+            } else {
+                // don't use console's __main__ so we can reuse the interpreter
+                jep.setInteractive(true);                
+                jep.set("jepInstance", jep);
+                jep.eval("from jep import console");
+                jep.eval("console.prompt(jepInstance)");
             }
-        catch(Throwable t) {
+        } catch (Throwable t) {
             t.printStackTrace();
-            if(jep != null)
+            if (jep != null)
                 jep.close();
 
             return 1;

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/Test.java

@@ -16,6 +16,7 @@ import jep.python.*;
  * @version $Id$
  */
 public class Test implements Runnable {
+    
     private Jep jep = null;
     private boolean testEval = false;
 
@@ -452,6 +453,45 @@ public class Test implements Runnable {
         return Thread.currentThread().getClass();
     }
     
+    public static void testRestrictedClassLoader() throws Throwable {
+        final Throwable[] t = new Throwable[1];
+        Thread thread = new Thread(new Runnable() {
+
+            @Override
+            public void run() {
+                Jep jep = null;
+                try {
+                    jep = new Jep(true, "", restrictedClassLoader);
+                    jep.eval("from java.io import File");                    
+                } catch (Throwable th) {
+                    t[0] = th;
+                } finally {
+                    if(jep != null) {
+                        jep.close();
+                    }
+                    synchronized (Test.class) {
+                        Test.class.notify();
+                    }
+                }
+            }
+        });
+        
+        synchronized (Test.class) {
+            thread.start();
+            try {
+                Test.class.wait();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+        
+        if(t[0] == null) {            
+            throw new RuntimeException("Did not throw classloader exception!");
+        } else if(!t[0].getMessage().contains("ImportError")) {
+            throw t[0];
+        }
+    }    
+    
     public static void main(String argv[]) throws Throwable {
         Jep jep = new Jep();
         try {

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/TestNumpy.java

@@ -4,8 +4,7 @@ import java.io.File;
 
 /**
  * TestNumpy.java. Runs a variety of simple tests to verify numpy interactions
- * are working correctly.  run() is called by setup.py test, or you can call the
+ * are working correctly.
- * java main() method directly for other tests.
  * 
  * 
  * Created: Wed Apr 08 2015
@@ -13,38 +12,65 @@ import java.io.File;
  * @author [ndjensen at gmail.com] Nate Jensen
  * @version $Id$
  */
-public class TestNumpy implements Runnable {
+public class TestNumpy {
-
-    protected Jep jep = null;
 
     // set to a high number to test for memory leaks
     private static final int REPEAT = 1; // 0000000;
 
-    private static boolean PRINT = false;
+    private static boolean PRINT = true;
+    
+    /**
+     * Calls testSetAndGet(Jep) on a separate thread to avoid threading
+     * issues.  Waits for those results before returning.
+     * @throws Throwable
+     */
+    public void testSetAndGet() throws Throwable
+    {
+        final Throwable[] t = new Throwable[1];
+        Thread thread = new Thread(new Runnable() {
 
             @Override
             public void run() {
+                Jep jep = null;
                 try {
-            File pwd = new File(".");
+                    jep = new Jep(true);
-            /*
+                                        
-             * Anytime you start a new Jep interpreter and import numpy within
+                } catch (Throwable th) {
-             * it, even if you call close() on the interpreter you will leak
+                    t[0] = th;
-             * some native memory. Therefore, for now, do NOT new up the
-             * interpreter and close it within the for loop.
-             */
-            jep = new Jep(false, pwd.getAbsolutePath());
-            for (int i = 0; i < REPEAT; i++) {
-                testSetAndGet();
-            }
-        } catch (JepException e) {
-            e.printStackTrace();
                 } finally {
-            if (jep != null)
+                    if(jep != null) {
                         jep.close();
                     }
+                    synchronized (TestNumpy.this) {
+                        TestNumpy.this.notify();
+                    }
+                }
+            }
+        });
+        
+        synchronized (TestNumpy.this) {
+            thread.start();
+            try {
+                TestNumpy.this.wait();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         }
         
-    public void testSetAndGet() throws JepException {
+        if(t[0] != null) {
+            throw t[0];
+        }
+
+    }
+
+    /**
+     * Sets NDArrays in a Jep interpreter, then gets them and verifies
+     * the conversion in both directions is safe, ie produces a symmetrical
+     * object despite a different reference/instance.
+     * @param jep
+     * @throws JepException
+     */
+    public void testSetAndGet(Jep jep) throws JepException {
         int[] dimensions = new int[] { 4 };
 
         // test boolean[]
@@ -178,6 +204,12 @@ public class TestNumpy implements Runnable {
         }
     }
 
+    /**
+     * Called from python to verify that a Java method's return type of NDArray
+     * can be auto-converted to a numpy ndarray.
+     * @param array
+     * @return a copy of the data + 5
+     */
     public NDArray<int[]> testArgAndReturn(NDArray<int[]> array) {
         int[] data = array.getData();
         int[] newData = new int[data.length];
@@ -188,29 +220,32 @@ public class TestNumpy implements Runnable {
         return new NDArray<int[]>(newData, array.getDimensions());
     }
 
-    // should only be called from main(), not from python unittests
+
-    public void runPythonSide() {
+    /**
-        try {
+     * Helper method to support running the main() method to run from Java
-            File pwd = new File("tests");
+     * instead of python
-            /*
+     * @param jep
-             * Anytime you start a new Jep interpreter and import numpy within
-             * it, even if you call close() on the interpreter you will leak
-             * some native memory. Therefore, for now, do NOT new up the
-             * interpreter and close it within the for loop.
      */
-            jep = new Jep(true, pwd.getAbsolutePath());
+    public void runFromJava(Jep jep) {
+        try {
             jep.eval("import test_numpy");
             jep.eval("v = test_numpy.TestNumpy('testArgReturn')");            
             jep.eval("v.setUp()");            
+            for (int i = 0; i < REPEAT; i++) {
+                this.testSetAndGet(jep);
+            }
             for (int i = 0; i < REPEAT; i++) {
                 jep.eval("v.testArgReturn()");
             }
+            System.out.println("return NDArray from Java checked out ok");
             for (int i = 0; i < REPEAT; i++) {
                 jep.eval("v.testMultiDimensional()");
             }
+            System.out.println("multi dimensional arrays checked out ok");
             for (int i = 0; i < REPEAT; i++) {
                 jep.eval("v.testArrayParams()");            
             }
+            System.out.println("Passing ndarrays to Java method as Java primitive[] checked out ok");
         } catch (JepException e) {
             e.printStackTrace();
         } finally {
@@ -219,34 +254,79 @@ public class TestNumpy implements Runnable {
         }
     }
 
+    /**
+     * Verifies a numpy.ndarray of bool can automatically convert to a method
+     * arg of boolean[]
+     * @param array
+     * @return true on success
+     */
     public boolean callBooleanMethod(boolean[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of byte can automatically convert to a method
+     * arg of byte[]
+     * @param array
+     * @return true on success
+     */
     public boolean callByteMethod(byte[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of int16 can automatically convert to a method
+     * arg of short[]
+     * @param array
+     * @return true on success
+     */
     public boolean callShortMethod(short[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of int32 can automatically convert to a method
+     * arg of int[]
+     * @param array
+     * @return true on success
+     */
     public boolean callIntMethod(int[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of int64 can automatically convert to a method
+     * arg of long[]
+     * @param array
+     * @return true on success
+     */
     public boolean callLongMethod(long[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of float32 can automatically convert to a method
+     * arg of float[]
+     * @param array
+     * @return true on success
+     */
     public boolean callFloatMethod(float[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies a numpy.ndarray of float64 can automatically convert to a method
+     * arg of double[]
+     * @param array
+     * @return true on success
+     */
     public boolean callDoubleMethod(double[] array) {
         return array != null;
     }
 
+    /**
+     * Verifies that an NDArray will not allow bad/dangerous constructor args.
+     */
     public void testNDArraySafety() {
         float[][] f = new float[15][];
         int[] dims = new int[] { 15, 20 };
@@ -257,7 +337,7 @@ public class TestNumpy implements Runnable {
                     "NDArray should have failed instantiation");
         } catch (IllegalArgumentException e) {
             if (PRINT) {
-                e.printStackTrace();
+                System.out.println("NDArray blocked bad type args");
             }
         }
 
@@ -269,7 +349,7 @@ public class TestNumpy implements Runnable {
                     "NDArray should have failed instantiation");
         } catch (IllegalArgumentException e) {
             if (PRINT) {
-                e.printStackTrace();
+                System.out.println("NDArray blocked bad dimensions args");
             }
         }
     }
@@ -281,10 +361,21 @@ public class TestNumpy implements Runnable {
      * @param args
      */
     public static void main(String[] args) {        
-        TestNumpy test = new TestNumpy();
+        File pwd = new File("tests");
-        test.run();
+        TestNumpy test = null;
-        test.runPythonSide();
+        Jep jep = null;
+        try {
+            test = new TestNumpy();
+            jep = new Jep(false, pwd.getPath());
+            test.runFromJava(jep);
             test.testNDArraySafety();
+        } catch (JepException e) {
+            e.printStackTrace();
+        } finally {
+            if (jep != null) {
+                jep.close();
+            }
+        }
     }
 
 }

nativeLib/rary.cots.jepp/jepp-2.3/src/jep/TestNumpyGILFreeze.java

@@ -4,10 +4,9 @@ package jep;
  * A test class that illustrates how numpy's floating point error handling can
  * cause python to deadlock trying to acquire the GIL. More specifically,
  * ufunc_object.c calls NPY_ALLOW_C_API which attempts to acquire the GIL.
- * Python is confused about the thread state and GIL state due to the top level
+ * Python is confused about the thread state and GIL state due to more than
- * interpreter being initialized on the same thread as the sub-interpreter(s).
+ * one Jep interpreter on the same thread.
  * 
- * Please see the TODO below for how to trigger the freeze.
  * 
  * Created: Thu Apr 09 2015
  * 
@@ -29,20 +28,11 @@ public class TestNumpyGILFreeze {
      * @param args
      */
     public static void main(String[] args) {   
-        /*
-         * TODO: To show the freeze, you need to alter Jep.TopInterpreter to
-         * NOT use a separate thread for System.loadLibrary("jep");
-         */
-        createJepAndUseNumpy();
-        createJepAndUseNumpy();
-
-        System.out.println("java main() finished");
-    }
-
-    public static void createJepAndUseNumpy() {
         Jep jep = null;
         try {
+            Jep jep0 = new Jep(true);
             jep = new Jep(true);
+            jep0.close();
             jep.eval("import numpy");
             /*
              * If error conditions are set to ignore, we will not reach the
@@ -51,7 +41,7 @@ public class TestNumpyGILFreeze {
              */
             jep.eval("numpy.seterr(under='print')");
             jep.eval(UNDERFLOW);
-            jep.eval("forceUnderflow()");
+            jep.eval("forceUnderflow()"); // this line will freeze
             System.out.println("returned from python interpreter");
         } catch (JepException e) {
             e.printStackTrace();
@@ -61,6 +51,7 @@ public class TestNumpyGILFreeze {
                 jep.close();
             }
         }
+        System.out.println("java main() finished");
     }
 
 }