xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Omaha #3676 harden numpy array to java array conversions to work with newer versions of python/numpy and not crash when heap cannot be allocated" into omaha_14.4.1

Browse source 
Former-commit-id: bb418e47ed [formerly bb418e47ed [formerly 57052aea644edc471adc3961739e07d4dcf80197]]
Former-commit-id: 21042a8945
Former-commit-id: 74086cdd1a
This commit is contained in:
Greg Armendariz 2014-10-15 14:19:28 -05:00 • committed by Gerrit Code Review
commit 7f2f8378ba
5 changed files with 185 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/BaseGfePyController.java
View file

@ -19,12 +19,17 @@
**/
package com.raytheon.viz.gfe;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import jep.JepException;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord.GridType;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat;
import com.raytheon.uf.common.python.controller.PythonScriptController;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
@ -43,6 +48,9 @@ import com.raytheon.viz.gfe.smartscript.FieldDefinition;
* Nov 10, 2008 njensen Initial creation
* Jan 08, 2013 1486 dgilling Refactor based on PythonScriptController.
* Jan 18, 2013 njensen Added garbageCollect()
* Oct 14, 2014 3676 njensen Moved getNumpyResult(GridType) here and
* hardened it by separating jep.getValue()
* calls from python copying/casting to correct types
*
* </pre>
*
@ -179,4 +187,114 @@ public abstract class BaseGfePyController extends PythonScriptController {
"Error garbage collecting GFE python interpreter", e);
}
}
/**
* Transforms the execution result of a python GFE script to a type expected
* based on the GridType. Currently used by smart tools and VC modules.
*
* @param type
* the type of data that is coming back
* @return the result of the execution in Java format
* @throws JepException
*/
protected Object getNumpyResult(GridType type) throws JepException {
Object result = null;
boolean resultFound = (Boolean) jep.getValue(RESULT + " is not None");
if (resultFound) {
int xDim, yDim = 0;
/*
* correctType is just a memory optimization. A copy is made when we
* call getValue(numpyArray), but doing array.astype(dtype) or
* numpy.ascontiguousarray(array, dtype) will create yet another
* copy.
*
* Note that if you attempt jep.getValue(array.astype(dtype)) or
* jep.getValue(numpy.ascontiguousarray(array, dtype)) you can
* potentially crash the JVM. jep.getValue(variable) should
* primarily retrieve variables that are globally scoped in the
* python interpreter as opposed to created on the fly.
*/
boolean correctType = false;
switch (type) {
case SCALAR:
correctType = (boolean) jep.getValue(RESULT
+ ".dtype.name == 'float32'");
if (!correctType) {
/*
* the following line needs to be separate from
* jep.getValue() to be stable
*/
jep.eval(RESULT + " = numpy.ascontiguousarray(" + RESULT
+ ", numpy.float32)");
}
float[] scalarData = (float[]) jep.getValue(RESULT);
xDim = (Integer) jep.getValue(RESULT + ".shape[1]");
yDim = (Integer) jep.getValue(RESULT + ".shape[0]");
result = new Grid2DFloat(xDim, yDim, scalarData);
break;
case VECTOR:
correctType = (boolean) jep.getValue(RESULT
+ "[0].dtype.name == 'float32'");
if (!correctType) {
/*
* the following line needs to be separate from
* jep.getValue() to be stable
*/
jep.eval(RESULT + "[0] = numpy.ascontiguousarray(" + RESULT
+ "[0], numpy.float32)");
}
correctType = (boolean) jep.getValue(RESULT
+ "[1].dtype.name == 'float32'");
if (!correctType) {
/*
* the following line needs to be separate from
* jep.getValue() to be stable
*/
jep.eval(RESULT + "[1] = numpy.ascontiguousarray(" + RESULT
+ "[1], numpy.float32)");
}
float[] mag = (float[]) jep.getValue(RESULT + "[0]");
float[] dir = (float[]) jep.getValue(RESULT + "[1]");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DFloat magGrid = new Grid2DFloat(xDim, yDim, mag);
Grid2DFloat dirGrid = new Grid2DFloat(xDim, yDim, dir);
result = new Grid2DFloat[] { magGrid, dirGrid };
break;
case WEATHER:
case DISCRETE:
correctType = (boolean) jep.getValue(RESULT
+ "[0].dtype.name == 'int8'");
if (!correctType) {
/*
* the following line needs to be separate from
* jep.getValue() to be stable
*/
jep.eval(RESULT + "[0] = numpy.ascontiguousarray(" + RESULT
+ "[0], numpy.int8)");
}
byte[] bytes = (byte[]) jep.getValue(RESULT + "[0]");
String[] keys = (String[]) jep.getValue(RESULT + "[1]");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DByte grid = new Grid2DByte(xDim, yDim, bytes);
List<String> keysList = new ArrayList<String>();
Collections.addAll(keysList, keys);
result = new Object[] { grid, keysList };
break;
}
jep.eval(RESULT + " = None");
}
return result;
}
}

56
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/core/parm/vcparm/VCModuleController.java
View file

@ -21,15 +21,12 @@ package com.raytheon.viz.gfe.core.parm.vcparm;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import jep.JepException;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord.GridType;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat;
import com.raytheon.uf.common.dataplugin.gfe.python.GfePyIncludeUtil;
import com.raytheon.uf.common.python.PyConstants;
import com.raytheon.viz.gfe.BaseGfePyController;
@ -48,6 +45,8 @@ import com.raytheon.viz.gfe.core.DataManager;
* ------------ ---------- ----------- --------------------------
* Nov 17, 2011 dgilling Initial creation
* Jan 08, 2013 1486 dgilling Support changes to BaseGfePyController.
* Oct 14, 2014 3676 njensen Removed decodeGD(GridType) since it was
* a copy of getNumpyResult()
*
* </pre>
*
@ -119,10 +118,11 @@ public class VCModuleController extends BaseGfePyController {
if (!methodName.equals("calcGrid")) {
jep.eval(RESULT + " = JUtil.pyValToJavaObj(" + RESULT + ")");
obj = jep.getValue(RESULT);
jep.eval(RESULT + " = None");
} else {
obj = decodeGD(type);
obj = getNumpyResult(type);
// getNumpyResult will set result to None to free up memory
}
jep.eval(RESULT + " = None");
return obj;
}
@ -169,52 +169,6 @@ public class VCModuleController extends BaseGfePyController {
tempGridNames.clear();
}
private Object decodeGD(GridType type) throws JepException {
Object result = null;
boolean resultFound = (Boolean) jep.getValue(RESULT + " is not None");
if (resultFound) {
int xDim, yDim = 0;
switch (type) {
case SCALAR:
float[] scalarData = (float[]) jep.getValue(RESULT
+ ".astype(numpy.float32)");
xDim = (Integer) jep.getValue(RESULT + ".shape[1]");
yDim = (Integer) jep.getValue(RESULT + ".shape[0]");
result = new Grid2DFloat(xDim, yDim, scalarData);
break;
case VECTOR:
float[] mag = (float[]) jep.getValue(RESULT
+ "[0].astype(numpy.float32)");
float[] dir = (float[]) jep.getValue(RESULT
+ "[1].astype(numpy.float32)");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DFloat magGrid = new Grid2DFloat(xDim, yDim, mag);
Grid2DFloat dirGrid = new Grid2DFloat(xDim, yDim, dir);
result = new Grid2DFloat[] { magGrid, dirGrid };
break;
case WEATHER:
case DISCRETE:
byte[] bytes = (byte[]) jep.getValue(RESULT
+ "[0].astype(numpy.int8)");
String[] keys = (String[]) jep.getValue(RESULT + "[1]");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DByte grid = new Grid2DByte(xDim, yDim, bytes);
List<String> keysList = new ArrayList<String>();
Collections.addAll(keysList, keys);
result = new Object[] { grid, keysList };
break;
}
}
return result;
}
/*
* (non-Javadoc)
*

66
cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/smarttool/script/SmartToolController.java
View file

@ -21,7 +21,6 @@ package com.raytheon.viz.gfe.smarttool.script;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -29,9 +28,7 @@ import java.util.Set;
import jep.JepException;
import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord.GridType;
import com.raytheon.uf.common.dataplugin.gfe.discrete.DiscreteKey;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DByte;
import com.raytheon.uf.common.dataplugin.gfe.grid.Grid2DFloat;
import com.raytheon.uf.common.dataplugin.gfe.python.GfePyIncludeUtil;
import com.raytheon.uf.common.dataplugin.gfe.weather.WeatherKey;
@ -69,8 +66,10 @@ import com.raytheon.viz.gfe.core.wxvalue.WxValue;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 20, 2008 njensen Initial creation
* Oct 29, 2013 2476 njensen Renamed numeric methods to numpy
* 10/31/2013 2508 randerso Change to use DiscreteGridSlice.getKeys()
* Oct 29, 2013 2476 njensen Renamed numeric methods to numpy
* 10/31/2013 2508 randerso Change to use DiscreteGridSlice.getKeys()
* Oct 14, 2014 3676 njensen Promoted getNumpyResult() to parent class
*
* </pre>
*
* @author njensen
@ -239,63 +238,6 @@ public class SmartToolController extends BaseGfePyController {
return (String) execute("getWeatherElementEdited", INTERFACE, args);
}
/**
* Transforms the execution result of a smart tool to a type that can be
* handled by the GridCycler
*
* @param type
* the type of data that is coming back from the smart tool
* @return the result of the execution in Java format
* @throws JepException
*/
protected Object getNumpyResult(GridType type) throws JepException {
Object result = null;
boolean resultFound = (Boolean) jep.getValue(RESULT + " is not None");
if (resultFound) {
int xDim, yDim = 0;
switch (type) {
case SCALAR:
float[] scalarData = (float[]) jep.getValue(RESULT
+ ".astype(numpy.float32)");
xDim = (Integer) jep.getValue(RESULT + ".shape[1]");
yDim = (Integer) jep.getValue(RESULT + ".shape[0]");
result = new Grid2DFloat(xDim, yDim, scalarData);
break;
case VECTOR:
float[] mag = (float[]) jep.getValue(RESULT
+ "[0].astype(numpy.float32)");
float[] dir = (float[]) jep.getValue(RESULT
+ "[1].astype(numpy.float32)");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DFloat magGrid = new Grid2DFloat(xDim, yDim, mag);
Grid2DFloat dirGrid = new Grid2DFloat(xDim, yDim, dir);
result = new Grid2DFloat[] { magGrid, dirGrid };
break;
case WEATHER:
case DISCRETE:
byte[] bytes = (byte[]) jep.getValue(RESULT
+ "[0].astype(numpy.int8)");
String[] keys = (String[]) jep.getValue(RESULT + "[1]");
xDim = (Integer) jep.getValue(RESULT + "[0].shape[1]");
yDim = (Integer) jep.getValue(RESULT + "[0].shape[0]");
Grid2DByte grid = new Grid2DByte(xDim, yDim, bytes);
List<String> keysList = new ArrayList<String>();
Collections.addAll(keysList, keys);
result = new Object[] { grid, keysList };
break;
}
jep.eval(RESULT + " = None");
}
return result;
}
/**
* Evaluates python method arguments for smart tools, transforming Java
* objects into python objects where appropriate.

7
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/pyembed.c
View file

@ -1128,12 +1128,7 @@ jobject pyembed_box_py(JNIEnv *env, PyObject *result) {
// added by njensen
init();
if(PyArray_Check(result)) {
jarray arr = NULL;
arr = numpyToJavaArray(env, result, NULL);
if(arr != NULL)
return arr;
return numpyToJavaArray(env, result, NULL);
}
// convert everything else to string

129
nativeLib/rary.cots.jepp/jepp-2.3/src/jep/util.c
View file

@ -1732,104 +1732,89 @@ jvalue convert_pyarg_jvalue(JNIEnv *env,
jarray numpyToJavaArray(JNIEnv* env, PyObject *param, jclass desiredType)
{
int sz;
enum NPY_TYPES paType;
jarray arr = NULL;
PyObject *nobj = NULL;
PyObject *nvalue = NULL;
PyArrayObject *pa = NULL;
int minDim = 0;
int maxDim = 0;
PyArrayObject *copy = NULL;
initNumpy();
// determine what we can about the pyarray that is to be converted
sz = PyArray_Size(param);
paType = ((PyArrayObject *) param)->descr->type_num;
if(desiredType == NULL)
{
if(((PyArrayObject *) param)->descr->type_num == NPY_BOOL)
if(paType == NPY_BOOL)
desiredType = JBOOLEAN_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_BYTE)
else if(paType == NPY_BYTE)
desiredType = JBYTE_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_INT16)
else if(paType == NPY_INT16)
desiredType = JSHORT_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_INT32)
else if(paType == NPY_INT32)
desiredType = JINT_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_INT64)
else if(paType == NPY_INT64)
desiredType = JLONG_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_FLOAT32)
else if(paType == NPY_FLOAT32)
desiredType = JFLOAT_ARRAY_TYPE;
else if(((PyArrayObject *) param)->descr->type_num == NPY_FLOAT64)
else if(paType == NPY_FLOAT64)
desiredType = JDOUBLE_ARRAY_TYPE;
}
if(desiredType != NULL)
{
if((*env)->IsSameObject(env, desiredType, JBOOLEAN_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_BOOL))
{
copy = (PyArrayObject *) PyArray_CopyFromObject(param, paType, 0, 0);
if((*env)->IsSameObject(env, desiredType, JBOOLEAN_ARRAY_TYPE) && (paType == NPY_BOOL)) {
arr = (*env)->NewBooleanArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_BOOL, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_BOOL);
pa = (PyArrayObject *) nvalue;
(*env)->SetBooleanArrayRegion(env, arr, 0, sz, (const jboolean *)pa->data);
}
else if((*env)->IsSameObject(env, desiredType, JBYTE_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_BYTE))
{
arr = (*env)->NewByteArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_BYTE, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_BYTE);
pa = (PyArrayObject *) nvalue;
(*env)->SetByteArrayRegion(env, arr, 0, sz, (const jbyte *)pa->data);
}
else if((*env)->IsSameObject(env, desiredType, JSHORT_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_INT16))
{
arr = (*env)->NewShortArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_INT16, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_INT16);
pa = (PyArrayObject *) nvalue;
(*env)->SetShortArrayRegion(env, arr, 0, sz, (const jshort *)pa->data);
}
else if((*env)->IsSameObject(env, desiredType, JINT_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_INT32))
{
else if((*env)->IsSameObject(env, desiredType, JBYTE_ARRAY_TYPE) && (paType == NPY_BYTE)) {
arr = (*env)->NewByteArray(env, sz);
}
else if((*env)->IsSameObject(env, desiredType, JSHORT_ARRAY_TYPE) && (paType == NPY_INT16)) {
arr = (*env)->NewShortArray(env, sz);
}
else if((*env)->IsSameObject(env, desiredType, JINT_ARRAY_TYPE) && (paType == NPY_INT32)) {
arr = (*env)->NewIntArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_INT32, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_INT32);
pa = (PyArrayObject *) nvalue;
(*env)->SetIntArrayRegion(env, arr, 0, sz, (const jint *)pa->data);
}
else if((*env)->IsSameObject(env, desiredType, JLONG_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_INT64))
{
}
else if((*env)->IsSameObject(env, desiredType, JLONG_ARRAY_TYPE) && (paType == NPY_INT64)) {
arr = (*env)->NewLongArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_INT64, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_INT64);
pa = (PyArrayObject *) nvalue;
(*env)->SetLongArrayRegion(env, arr, 0, sz, (const jlong *)pa->data);
}
else if ((*env)->IsSameObject(env, desiredType, JFLOAT_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_FLOAT32))
{
}
else if ((*env)->IsSameObject(env, desiredType, JFLOAT_ARRAY_TYPE) && (paType == NPY_FLOAT32)) {
arr = (*env)->NewFloatArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_FLOAT32, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_FLOAT32);
pa = (PyArrayObject *) nvalue;
(*env)->SetFloatArrayRegion(env, arr, 0, sz, (const jfloat *)pa->data);
}
else if((*env)->IsSameObject(env, desiredType, JDOUBLE_ARRAY_TYPE)
&& (((PyArrayObject *) param)->descr->type_num == NPY_FLOAT64))
{
}
else if((*env)->IsSameObject(env, desiredType, JDOUBLE_ARRAY_TYPE) && (paType == NPY_FLOAT64)) {
arr = (*env)->NewDoubleArray(env, sz);
nobj = PyArray_ContiguousFromObject(param, NPY_FLOAT64, minDim, maxDim);
nvalue = (PyObject *)PyArray_Cast((PyArrayObject *)nobj, NPY_FLOAT64);
pa = (PyArrayObject *) nvalue;
(*env)->SetDoubleArrayRegion(env, arr, 0, sz, (const jdouble *)pa->data);
}
// java exception could potentially be OutOfMemoryError if it couldn't allocate the array
if(process_java_exception(env) || arr == NULL) {
return NULL;
}
// if arr was allocated, we already know it matched the python array type
if(paType == NPY_BOOL) {
(*env)->SetBooleanArrayRegion(env, arr, 0, sz, (const jboolean *)copy->data);
}
else if(paType == NPY_BYTE) {
(*env)->SetByteArrayRegion(env, arr, 0, sz, (const jbyte *)copy->data);
}
else if(paType == NPY_INT16) {
(*env)->SetShortArrayRegion(env, arr, 0, sz, (const jshort *)copy->data);
}
else if(paType == NPY_INT32) {
(*env)->SetIntArrayRegion(env, arr, 0, sz, (const jint *)copy->data);
}
else if(paType == NPY_INT64) {
(*env)->SetLongArrayRegion(env, arr, 0, sz, (const jlong *)copy->data);
}
else if(paType == NPY_FLOAT32) {
(*env)->SetFloatArrayRegion(env, arr, 0, sz, (const jfloat *)copy->data);
}
else if(paType == NPY_FLOAT64) {
(*env)->SetDoubleArrayRegion(env, arr, 0, sz, (const jdouble *)copy->data);
}
}
if(nobj != NULL)
Py_DECREF(nobj);
if(pa != NULL)
Py_DECREF(pa);
if(copy != NULL)
Py_DECREF(copy);
return arr;
}