dataaccess checkout from 17.3.1

edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightningAccessFactory.java

@@ -23,6 +23,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -30,15 +31,16 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.raytheon.uf.common.dataaccess.IDataRequest;
 import com.raytheon.uf.common.dataaccess.exception.IncompatibleRequestException;
+import com.raytheon.uf.common.dataaccess.exception.TimeAgnosticDataException;
 import com.raytheon.uf.common.dataaccess.geom.IGeometryData;
 import com.raytheon.uf.common.dataaccess.geom.IGeometryData.Type;
 import com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory;
 import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData;
 import com.raytheon.uf.common.dataaccess.util.PDOUtil;
-import com.raytheon.uf.common.dataplugin.PluginDataObject;
 import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord;
 import com.raytheon.uf.common.dataplugin.binlightning.LightningConstants;
 import com.raytheon.uf.common.dataquery.requests.RequestConstraint;
@@ -54,6 +56,7 @@ import com.raytheon.uf.common.datastorage.records.IntegerDataRecord;
 import com.raytheon.uf.common.datastorage.records.LongDataRecord;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.time.BinOffset;
 import com.raytheon.uf.common.time.DataTime;
 import com.vividsolutions.jts.geom.Coordinate;
 import com.vividsolutions.jts.geom.Envelope;
@@ -83,6 +86,9 @@ import com.vividsolutions.jts.geom.GeometryFactory;
  * Jun 07, 2016  5587     tgurney   Change get*Identifiers() to take
  *                                  IDataRequest
  * Aug 01, 2016  2416     tgurney   Add dataURI as optional identifier
+ * Nov 30, 2016  6018     tgurney   Filter out keep-alive records from available
+ *                                  times
+ * Mar 06, 2017  6142     bsteffen  Remove dataURI as optional identifier
  *
  * </pre>
  *
@@ -105,7 +111,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
 
     private static final String[] requiredKeys = { timeKey, latKey, lonKey };
 
-    public static final String[] AVAILABLE_PARAMETERS = {
+    protected static final String[] AVAILABLE_PARAMETERS = {
             LightningConstants.INTENSITY_DATASET,
             LightningConstants.MSG_TYPE_DATASET,
             LightningConstants.STRIKE_TYPE_DATASET,
@@ -116,8 +122,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
 
     @Override
     public String[] getAvailableLocationNames(IDataRequest request) {
-        throw new IncompatibleRequestException(this.getClass()
+        throw new IncompatibleRequestException(
-                + " does not support location names");
+                this.getClass() + " does not support location names");
     }
 
     @Override
@@ -130,11 +136,6 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
         return new String[] { sourceKey };
     }
 
-    @Override
-    public String[] getOptionalIdentifiers(IDataRequest request) {
-        return new String[] { PluginDataObject.DATAURI_ID };
-    }
-
     @Override
     protected Map<String, RequestConstraint> buildConstraintsFromRequest(
             IDataRequest request) {
@@ -158,7 +159,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
     @Override
     protected IGeometryData[] getGeometryData(IDataRequest request,
             DbQueryResponse dbQueryResponse) {
-        Map<File, List<BinLightningRecord>> results = unpackResults(dbQueryResponse);
+        Map<File, List<BinLightningRecord>> results = unpackResults(
+                dbQueryResponse);
 
         List<IGeometryData> rval = new ArrayList<>();
         for (Entry<File, List<BinLightningRecord>> resultEntry : results
@@ -224,8 +226,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
                         .getFloatData();
 
                 for (int i = 0; i < timeData.length; i++) {
-                    if (envelope != null
+                    if (envelope != null && !envelope.contains(longitudeData[i],
-                            && !envelope.contains(longitudeData[i],
                             latitudeData[i])) {
                         /* Skip any data the user doesn't want */
                         continue;
@@ -234,8 +235,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
                     DefaultGeometryData data = new DefaultGeometryData();
                     data.setDataTime(dt);
                     data.addAttribute(sourceKey, source);
-                    data.setGeometry(geomFactory.createPoint(new Coordinate(
+                    data.setGeometry(geomFactory.createPoint(
-                            longitudeData[i], latitudeData[i])));
+                            new Coordinate(longitudeData[i], latitudeData[i])));
                     // add the optional parameter records
                     addParameterData(data, recordMap, k, i);
                     dataList.add(data);
@@ -268,7 +269,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
             String parameterName = entry.getKey();
             IDataRecord record = entry.getValue().get(recordIndex);
             if (record instanceof IntegerDataRecord) {
-                int value = ((IntegerDataRecord) record).getIntData()[valueIndex];
+                int value = ((IntegerDataRecord) record)
+                        .getIntData()[valueIndex];
                 data.addData(parameterName, value, Type.INT);
             } else if (record instanceof ByteDataRecord) {
                 int value = ((ByteDataRecord) record).getByteData()[valueIndex];
@@ -323,8 +325,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
             if (availableParams.contains(param)) {
                 included.add(param);
             } else {
-                throw new IncompatibleRequestException(param
+                throw new IncompatibleRequestException(
-                        + " is not a valid parameter for this request");
+                        param + " is not a valid parameter for this request");
             }
         }
         return new ArrayList<>(included);
@@ -365,4 +367,19 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory {
         return getAvailableValues(request, identifierKey, String.class);
     }
 
+    private boolean isKeepAlive(DataTime t) {
+        Calendar cal = t.getRefTimeAsCalendar();
+        return t.getValidPeriod().getStart().equals(t.getValidPeriod().getEnd())
+                && cal.get(Calendar.SECOND) == 0
+                && cal.get(Calendar.MILLISECOND) == 0;
+    }
+
+    @Override
+    public DataTime[] getAvailableTimes(IDataRequest request,
+            BinOffset binOffset) throws TimeAgnosticDataException {
+        DataTime[] times = super.getAvailableTimes(request, binOffset);
+        return Arrays.stream(times).filter(t -> !isKeepAlive(t))
+                .collect(Collectors.toList()).toArray(new DataTime[0]);
+    }
+
 }

edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java

@@ -47,11 +47,11 @@ import com.raytheon.uf.common.serialization.comm.RequestRouter;
  *                                     Browser, Volume Browser, and Data Access
  *                                     Framework.
  * Jul 01, 2014 3149      randerso     Changed to use updated GetGridRequest
+ * Dec 15, 2016 6040      tgurney      Added DB_TYPE constant
  *
  * </pre>
  *
  * @author bsteffen
- * @version 1.0
  */
 public class GFEDataAccessUtil {
 
@@ -71,6 +71,8 @@ public class GFEDataAccessUtil {
 
     public static final String PARM_LEVEL = PARM_ID + ".parmLevel";
 
+    public static final String DB_TYPE = DB_ID + ".dbType";
+
     /**
      * Retrieve the GridParmInfo for a ParmID
      *

edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEGridFactory.java

@@ -88,6 +88,9 @@ import com.raytheon.uf.common.util.StringUtil;
  *                                  IDataRequest
  * Jun 13, 2016  5574     mapeters  Add advanced query support
  * Aug 01, 2016  2416     tgurney   Add dataURI as optional identifier
+ * Dec 15, 2016  6040     tgurney   Add dbType as optional identifier
+ * Mar 06, 2017  6142     bsteffen  Remove dataURI as optional identifier
+ * 
  *
  * </pre>
  *
@@ -109,7 +112,7 @@ public class GFEGridFactory extends AbstractGridDataPluginFactory {
     private static final String[] OPTIONAL_IDENTIFIERS = {
             GFEDataAccessUtil.MODEL_NAME, GFEDataAccessUtil.MODEL_TIME,
             GFEDataAccessUtil.SITE_ID, MODEL_NAME, MODEL_TIME, SITE_ID,
-            PluginDataObject.DATAURI_ID };
+            GFEDataAccessUtil.DB_TYPE };
 
     @Override
     public String[] getOptionalIdentifiers(IDataRequest request) {
@@ -339,10 +342,10 @@ public class GFEGridFactory extends AbstractGridDataPluginFactory {
     @Override
     public String[] getIdentifierValues(IDataRequest request,
             String identifierKey) {
-        if (!Arrays.asList(getRequiredIdentifiers(request)).contains(
+        if (!Arrays.asList(getRequiredIdentifiers(request))
-                identifierKey)
+                .contains(identifierKey)
-                && !Arrays.asList(getOptionalIdentifiers(request)).contains(
+                && !Arrays.asList(getOptionalIdentifiers(request))
-                        identifierKey)) {
+                        .contains(identifierKey)) {
             throw new InvalidIdentifiersException(request.getDatatype(), null,
                     Arrays.asList(new String[] { identifierKey }));
         }

edexOsgi/com.raytheon.uf.common.dataplugin.grid/src/com/raytheon/uf/common/dataplugin/grid/dataaccess/GridDataAccessFactory.java

@@ -105,6 +105,7 @@ import com.vividsolutions.jts.geom.GeometryFactory;
  *                                  IDataRequest
  * Jul 06, 2016  5728     mapeters  Add advanced query support
  * Aug 01, 2016  2416     tgurney   Add dataURI as optional identifier
+ * Mar 06, 2017  6142     bsteffen  Remove dataURI as optional identifier
  * 
  * </pre>
  * 
@@ -114,14 +115,18 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
 
     public static final String NAMESPACE = "namespace";
 
-    public static final String[] VALID_IDENTIFIERS = {
+    protected static final String[] VALID_IDENTIFIERS = {
             GridConstants.DATASET_ID, GridConstants.SECONDARY_ID,
             GridConstants.ENSEMBLE_ID, NAMESPACE,
             GridConstants.MASTER_LEVEL_NAME, GridConstants.LEVEL_ONE,
-            GridConstants.LEVEL_TWO, PluginDataObject.DATAURI_ID };
+            GridConstants.LEVEL_TWO };
 
     @Override
     public String[] getOptionalIdentifiers(IDataRequest request) {
+        return getGridOptionalIdentifiers();
+    }
+
+    public static String[] getGridOptionalIdentifiers() {
         return VALID_IDENTIFIERS;
     }
 
@@ -140,8 +145,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
             DAFGridQueryAssembler assembler = new DAFGridQueryAssembler();
 
             if (identifiers != null && identifiers.containsKey(NAMESPACE)) {
-                assembler.setNamespace(getStringIdentifier(identifiers,
+                assembler.setNamespace(
-                        NAMESPACE));
+                        getStringIdentifier(identifiers, NAMESPACE));
             }
 
             if (request.getParameters() != null) {
@@ -157,10 +162,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                 checkForLevelConflict(request.getLevels(), identifiers);
 
                 for (Level level : request.getLevels()) {
-                    assembler.setMasterLevelName(level.getMasterLevel()
+                    assembler.setMasterLevelName(
-                            .getName());
+                            level.getMasterLevel().getName());
-                    assembler.setLevelUnits(level.getMasterLevel()
+                    assembler.setLevelUnits(
-                            .getUnitString());
+                            level.getMasterLevel().getUnitString());
                     assembler.setLevelOneValue(level.getLevelonevalue());
                     assembler.setLevelTwoValue(level.getLeveltwovalue());
                     // TODO Theoretically merging these could end badly if there
@@ -200,9 +205,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                             identifiers, GridConstants.SECONDARY_ID));
                 }
                 if (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME)) {
-                    assembler
+                    assembler.setMasterLevelNameConstraint(
-                            .setMasterLevelNameConstraint(getIdentifierConstraint(
+                            getIdentifierConstraint(identifiers,
-                                    identifiers,
                                     GridConstants.MASTER_LEVEL_NAME));
                 }
                 if (identifiers.containsKey(GridConstants.LEVEL_ONE)) {
@@ -256,7 +260,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
             return (String) value;
         } else {
             throw new IncompatibleRequestException(
-                    "Only string identifier values are valid for '" + key + "'");
+                    "Only string identifier values are valid for '" + key
+                            + "'");
         }
     }
 
@@ -275,11 +280,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
      */
     private static void checkForLevelConflict(Level[] levels,
             Map<String, Object> identifiers) {
-        if (levels.length > 0
+        if (levels.length > 0 && identifiers != null
-                && identifiers != null
                 && (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME)
-                        || identifiers.containsKey(GridConstants.LEVEL_ONE) || identifiers
+                        || identifiers.containsKey(GridConstants.LEVEL_ONE)
-                            .containsKey(GridConstants.LEVEL_TWO))) {
+                        || identifiers.containsKey(GridConstants.LEVEL_TWO))) {
             throw new DataRetrievalException(
                     "Conflict between the request levels and request "
                             + "identifiers. Please set the levels either as"
@@ -300,8 +304,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
         for (Entry<String, RequestConstraint> sourceEntry : source.entrySet()) {
             String key = sourceEntry.getKey();
             RequestConstraint sourceConstraint = sourceEntry.getValue();
-            RequestConstraint targetConstraint = target.get(sourceEntry
+            RequestConstraint targetConstraint = target
-                    .getKey());
+                    .get(sourceEntry.getKey());
             if (targetConstraint == null) {
                 target.put(key, sourceConstraint);
             } else if (!sourceConstraint.equals(targetConstraint)) {
@@ -309,8 +313,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                 // TODO we don't necessarily want to always add. This could
                 // result in something like IN MB,FHAG,MB,MB,MB, but we also
                 // don't want to parse the in list all the time.
-                targetConstraint.addToConstraintValueList(sourceConstraint
+                targetConstraint.addToConstraintValueList(
-                        .getConstraintValue());
+                        sourceConstraint.getConstraintValue());
             }
         }
     }
@@ -319,14 +323,14 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
     protected IGridData constructGridDataResponse(IDataRequest request,
             PluginDataObject pdo, GridGeometry2D gridGeometry,
             DataSource dataSource) {
-        if (pdo instanceof GridRecord == false) {
+        if (pdo instanceof GridRecord) {
-            throw new DataRetrievalException(this.getClass().getSimpleName()
-                    + " cannot handle " + pdo.getClass().getSimpleName());
-        }
             GridRecord gridRecord = (GridRecord) pdo;
             return constructGridDataResponse(request, gridRecord, gridGeometry,
                     dataSource);
         }
+        throw new DataRetrievalException(this.getClass().getSimpleName()
+                + " cannot handle " + pdo.getClass().getSimpleName());
+    }
 
     public static IGridData constructGridDataResponse(IDataRequest request,
             GridRecord gridRecord, GridGeometry2D gridGeometry,
@@ -341,17 +345,16 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
             // perform reverse mappings so the parameters and levels that are
             // returned match exactly what was requested.
             String namespace = getStringIdentifier(identifiers, NAMESPACE);
-            List<String> requestParameters = Arrays.asList(request
+            List<String> requestParameters = Arrays
-                    .getParameters());
+                    .asList(request.getParameters());
             parameter = reverseResolveMapping(ParameterMapper.getInstance(),
                     parameter, namespace, requestParameters);
 
             if (identifiers.containsKey(GridConstants.DATASET_ID)) {
                 RequestConstraint requestedDatasetsConstraint = getIdentifierConstraint(
                         identifiers, GridConstants.DATASET_ID);
-                datasetId = reverseResolveMapping(
+                datasetId = reverseResolveMapping(DatasetIdMapper.getInstance(),
-                        DatasetIdMapper.getInstance(), datasetId, namespace,
+                        datasetId, namespace, requestedDatasetsConstraint);
-                        requestedDatasetsConstraint);
             }
             if (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME)) {
                 RequestConstraint requestedMasterLevelConstraint = getIdentifierConstraint(
@@ -448,8 +451,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
      */
     @Override
     public String[] getAvailableParameters(IDataRequest request) {
-        return getAvailableValues(request,
+        return getAvailableValues(request, GridConstants.PARAMETER_ABBREVIATION,
-                GridConstants.PARAMETER_ABBREVIATION, String.class);
+                String.class);
     }
 
     /**
@@ -478,8 +481,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                 dbQueryResponse.getNumResults());
         for (Entry<GridGeometryKey, Set<GridRecord>> entry : sortedRecords
                 .entrySet()) {
-            result.addAll(getGeometryData(request, entry.getKey(),
+            result.addAll(
-                    entry.getValue()));
+                    getGeometryData(request, entry.getKey(), entry.getValue()));
 
         }
         return result.toArray(new IGeometryData[0]);
@@ -519,10 +522,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
         DefaultGeometryData data = key.toGeometryData();
         DirectPosition2D llPoint = findResponsePoint(key.getGridGeometry(),
                 point.x, point.y);
-        data.setGeometry(new GeometryFactory().createPoint(new Coordinate(
+        data.setGeometry(new GeometryFactory()
-                llPoint.x, llPoint.y)));
+                .createPoint(new Coordinate(llPoint.x, llPoint.y)));
-        data.setLocationName(data.getLocationName() + "-" + point.x + ","
+        data.setLocationName(
-                + point.y);
+                data.getLocationName() + "-" + point.x + "," + point.y);
         Request request = Request.buildPointRequest(point);
         populateGeometryData(records, request,
                 new DefaultGeometryData[] { data });
@@ -542,15 +545,17 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                 .getWidth() * gridRange.getHeight())];
         GeometryFactory geometryFactory = new GeometryFactory();
         int index = 0;
-        for (int y = (int) gridRange.getMinY(); y < gridRange.getMaxY(); y += 1) {
+        for (int y = (int) gridRange.getMinY(); y < gridRange
-            for (int x = (int) gridRange.getMinX(); x < gridRange.getMaxX(); x += 1) {
+                .getMaxY(); y += 1) {
+            for (int x = (int) gridRange.getMinX(); x < gridRange
+                    .getMaxX(); x += 1) {
                 data[index] = key.toGeometryData();
                 DirectPosition2D llPoint = findResponsePoint(
                         key.getGridGeometry(), x, y);
                 data[index].setGeometry(geometryFactory
                         .createPoint(new Coordinate(llPoint.x, llPoint.y)));
-                data[index].setLocationName(data[index].getLocationName() + "-"
+                data[index].setLocationName(
-                        + x + "," + y);
+                        data[index].getLocationName() + "-" + x + "," + y);
                 index += 1;
             }
         }
@@ -563,10 +568,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
     @Override
     public String[] getIdentifierValues(IDataRequest request,
             String identifierKey) {
-        if (!Arrays.asList(getRequiredIdentifiers(request)).contains(
+        if (!Arrays.asList(getRequiredIdentifiers(request))
-                identifierKey)
+                .contains(identifierKey)
-                && !Arrays.asList(getOptionalIdentifiers(request)).contains(
+                && !Arrays.asList(getOptionalIdentifiers(request))
-                        identifierKey)) {
+                        .contains(identifierKey)) {
             throw new InvalidIdentifiersException(request.getDatatype(), null,
                     Arrays.asList(identifierKey));
         }
@@ -602,25 +607,26 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
                             .getFloatData();
                     if (rawArray.length != data.length) {
                         throw new DataRetrievalException(
-                                "Unexpected response of size "
+                                "Unexpected response of size " + rawArray.length
-                                        + rawArray.length
                                         + " when expected size is "
-                                        + data.length + " for record " + record);
+                                        + data.length + " for record "
+                                        + record);
                     }
                     for (int i = 0; i < data.length; i += 1) {
                         data[i].addData(parameter.getAbbreviation(),
                                 rawArray[i], parameter.getUnit());
                     }
                 } else {
-                    String type = dataRecord == null ? "null" : dataRecord
+                    String type = dataRecord == null ? "null"
-                            .getClass().getSimpleName();
+                            : dataRecord.getClass().getSimpleName();
                     throw new DataRetrievalException("Unexpected record type("
                             + type + ") for " + record);
                 }
             } catch (Exception e) {
                 throw new DataRetrievalException(
                         "Failed to retrieve the IDataRecord for GridRecord: "
-                                + record.toString(), e);
+                                + record.toString(),
+                        e);
             }
         }
     }
@@ -669,7 +675,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
             int maxX = (int) Math.round(testPoints[2]);
             int maxY = (int) Math.round(testPoints[3]);
             GridEnvelope2D gridRange = gridGeometry.getGridRange2D();
-            if (minX == maxX && minY == maxY && gridRange.contains(minX, minY)) {
+            if (minX == maxX && minY == maxY
+                    && gridRange.contains(minX, minY)) {
                 return new Point(minX, minY);
             } else {
                 return null;
@@ -696,8 +703,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
 
         private final int hashCode;
 
-        public GridGeometryKey(Level level, DataTime dataTime,
+        public GridGeometryKey(Level level, DataTime dataTime, String datasetId,
-                String datasetId, GridGeometry2D gridGeometry) {
+                GridGeometry2D gridGeometry) {
             this.level = level;
             this.dataTime = dataTime;
             this.datasetId = datasetId;
@@ -716,9 +723,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory {
         }
 
         public GridGeometryKey(GridRecord record) {
-            this(record.getLevel(), record.getDataTime(),
+            this(record.getLevel(), record.getDataTime(), record.getDatasetId(),
-                    record.getDatasetId(), record.getLocation()
+                    record.getLocation().getGridGeometry());
-                            .getGridGeometry());
 
         }
 

edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarDataAccessFactory.java

@@ -0,0 +1,852 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ *
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ *
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ *
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.dataplugin.radar.dataaccess;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.geotools.coverage.grid.GridGeometry2D;
+import org.geotools.geometry.jts.ReferencedEnvelope;
+import org.opengis.referencing.FactoryException;
+
+import com.raytheon.uf.common.dataaccess.IDataRequest;
+import com.raytheon.uf.common.dataaccess.exception.DataRetrievalException;
+import com.raytheon.uf.common.dataaccess.exception.EnvelopeProjectionException;
+import com.raytheon.uf.common.dataaccess.exception.IncompatibleRequestException;
+import com.raytheon.uf.common.dataaccess.exception.InvalidIdentifiersException;
+import com.raytheon.uf.common.dataaccess.geom.IGeometryData;
+import com.raytheon.uf.common.dataaccess.grid.IGridData;
+import com.raytheon.uf.common.dataaccess.impl.AbstractGridDataPluginFactory;
+import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData;
+import com.raytheon.uf.common.dataaccess.impl.DefaultGridData;
+import com.raytheon.uf.common.dataaccess.util.PDOUtil;
+import com.raytheon.uf.common.dataplugin.PluginDataObject;
+import com.raytheon.uf.common.dataplugin.level.Level;
+import com.raytheon.uf.common.dataplugin.level.MasterLevel;
+import com.raytheon.uf.common.dataplugin.radar.RadarRecord;
+import com.raytheon.uf.common.dataplugin.radar.projection.RadarProjectionFactory;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarDataRetriever;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarInfo;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarInfoDict;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarUtil;
+import com.raytheon.uf.common.dataquery.requests.DbQueryRequest;
+import com.raytheon.uf.common.dataquery.requests.RequestConstraint;
+import com.raytheon.uf.common.dataquery.requests.RequestConstraint.ConstraintType;
+import com.raytheon.uf.common.dataquery.responses.DbQueryResponse;
+import com.raytheon.uf.common.datastorage.DataStoreFactory;
+import com.raytheon.uf.common.datastorage.IDataStore;
+import com.raytheon.uf.common.datastorage.StorageException;
+import com.raytheon.uf.common.geospatial.util.SubGridGeometryCalculator;
+import com.raytheon.uf.common.localization.PathManagerFactory;
+import com.raytheon.uf.common.numeric.buffer.ByteBufferWrapper;
+import com.raytheon.uf.common.numeric.buffer.ShortBufferWrapper;
+import com.raytheon.uf.common.numeric.filter.FillValueFilter;
+import com.raytheon.uf.common.numeric.source.DataSource;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Envelope;
+
+/**
+ *
+ * A data factory for getting radar data from the metadata database. There are
+ * currently not any required identifiers.
+ *
+ * This class handles requests for both grid data (i.e., radial and raster
+ * products) and geometry data (i.e., graphics products).
+ *
+ * Radar does not return subgrids for request envelopes like other gridded
+ * types. Instead data for only icaos within the request envelope are returned
+ * and all data for the product is used. This is done because subgridding radial
+ * products is complex and this is not often what a caller actually wants.
+ *
+ *
+ * <pre>
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date          Ticket#  Engineer    Description
+ * ------------- -------- ----------- --------------------------
+ * Jan 23, 2013           bsteffen    Initial creation
+ * Feb 14, 2013  1614     bsteffen    Refactor data access framework to use
+ *                                    single request.
+ * Feb 04, 2014  2672     bsteffen    Enable requesting icaos within envelope.
+ * Jul 30, 2014  3184     njensen     Overrode optional identifiers
+ * Oct 28, 2014  3755     nabowle     Implement getAvailableParameters, handle
+ *                                    empty parameters, fix error message, and
+ *                                    handle dataless radial radars.
+ * Dec 18, 2014  3600     nabowle     Implement getAvailableLevels and add
+ *                                    optional identifiers to indicate what
+ *                                    fields are used for the level one and two
+ *                                    values.
+ * Feb 13, 2015  4124     mapeters    Inherits IDataFactory.
+ * Feb 27, 2015  4179     mapeters    Use AbstractDataPluginFactory.getAvailableValues().
+ * Apr 18, 2016  5587     tgurney     Implement getIdentifierValues()
+ * Jun 07, 2016  5587     tgurney     Change get*Identifiers() to take
+ *                                    IDataRequest
+ * Jun 08, 2016  5574     mapeters    Add advanced query support, throw exception for
+ *                                    invalid level field identifiers
+ * Aug 01, 2016  2416     tgurney     Add dataURI as optional identifier
+ * Aug 29, 2016  2671     tgurney     Add support for melting layer graphics
+ * Aug 31, 2016  2671     tgurney     Add mesocyclone support
+ * Sep 09, 2016  2671     tgurney     Add storm track (STI) support
+ * Sep 27, 2016  2671     tgurney     Add hail index support
+ * Sep 28, 2016  2671     tgurney     Add tornado vortex sig (TVS) support
+ * Mar 06, 2017  6142     bsteffen    Remove dataURI as optional identifier
+ *
+ * </pre>
+ *
+ * @author bsteffen
+ */
+public class RadarDataAccessFactory extends AbstractGridDataPluginFactory {
+
+    private static final transient IUFStatusHandler statusHandler = UFStatus
+            .getHandler(RadarDataAccessFactory.class);
+
+    private static final String PRODUCT_CODE = "productCode";
+
+    private static final String PRIMARY_ANGLE = "primaryElevationAngle";
+
+    private static final String TRUE_ANGLE = "trueElevationAngle";
+
+    private static final String ELEVATION_NUMBER = "elevationNumber";
+
+    private static final String ICAO = "icao";
+
+    private static final String LONGITUDE = "longitude";
+
+    private static final String LATITUDE = "latitude";
+
+    private static final String FORMAT = "format";
+
+    private static final String RADIAL_FORMAT = "Radial";
+
+    private static final String RASTER_FORMAT = "Raster";
+
+    private static final String GRAPHIC_FORMAT = "Graphic";
+
+    private static final String LEVEL_ONE = "level.one.field";
+
+    private static final String LEVEL_TWO = "level.two.field";
+
+    private static final List<String> SUPPORTED_GRID_FORMATS = Arrays
+            .asList(RADIAL_FORMAT, RASTER_FORMAT);
+
+    private static final List<String> SUPPORTED_GEOMETRY_FORMATS = Arrays
+            .asList(GRAPHIC_FORMAT);
+
+    private static final List<String> SUPPORTED_FORMATS = Arrays
+            .asList(RADIAL_FORMAT, RASTER_FORMAT, GRAPHIC_FORMAT);
+
+    private static final List<String> SUPPORTED_LEVELS = Arrays
+            .asList(PRIMARY_ANGLE, TRUE_ANGLE, ELEVATION_NUMBER);
+
+    private static final String LEVEL_ERROR = " must be " + PRIMARY_ANGLE + ", "
+            + TRUE_ANGLE + ", or " + ELEVATION_NUMBER;
+
+    private static RadarInfoDict radarInfo = null;
+
+    private static MasterLevel tiltLevel = null;
+
+    @Override
+    protected IGridData constructGridDataResponse(IDataRequest request,
+            PluginDataObject pdo, GridGeometry2D gridGeometry,
+            DataSource dataSource) {
+        RadarRecord radarRecord = asRadarRecord(pdo);
+        RadarInfo radarInfo = getRadarInfo()
+                .getInfo(radarRecord.getProductCode());
+        if (!SUPPORTED_GRID_FORMATS.contains(radarInfo.getFormat())) {
+            throw new DataRetrievalException(radarInfo.getFormat()
+                    + " data cannot be requested as grid");
+        }
+        DefaultGridData defaultGridData = new DefaultGridData(dataSource,
+                gridGeometry);
+        defaultGridData.setDataTime(pdo.getDataTime());
+        // reverse map parameter to match request.
+        List<String> requestedParameters = Arrays
+                .asList(request.getParameters());
+        if (requestedParameters.contains(radarInfo.getName())) {
+            defaultGridData.setParameter(radarInfo.getName());
+        } else if (requestedParameters.contains(radarInfo.getMnemonic())) {
+            defaultGridData.setParameter(radarRecord.getMnemonic());
+        } else {
+            defaultGridData
+                    .setParameter(radarRecord.getProductCode().toString());
+        }
+        defaultGridData.setUnit(radarRecord.getDataUnit());
+        defaultGridData.setLevel(getLevel(radarRecord, request));
+        defaultGridData.setLocationName(generateLocationName(radarRecord));
+
+        Map<String, Object> attributes = new HashMap<>();
+        attributes.put(ICAO, radarRecord.getIcao());
+        attributes.put(FORMAT, radarRecord.getFormat());
+
+        defaultGridData.setAttributes(attributes);
+
+        return defaultGridData;
+    }
+
+    @Override
+    protected IGeometryData[] getGeometryData(IDataRequest request,
+            DbQueryResponse dbQueryResponse) {
+
+        Map<File, List<RadarRecord>> results = unpackResults(dbQueryResponse);
+        List<IGeometryData> rval = new ArrayList<>();
+
+        for (Entry<File, List<RadarRecord>> resultEntry : results.entrySet()) {
+            IDataStore ds = DataStoreFactory.getDataStore(resultEntry.getKey());
+            for (RadarRecord radarRecord : resultEntry.getValue()) {
+                RadarInfo radarInfo = getRadarInfo()
+                        .getInfo(radarRecord.getProductCode());
+
+                if (!SUPPORTED_GEOMETRY_FORMATS
+                        .contains(radarInfo.getFormat())) {
+                    throw new DataRetrievalException(radarInfo.getFormat()
+                            + " data cannot be requested as geometry");
+                }
+
+                try {
+                    RadarDataRetriever.populateRadarRecord(ds, radarRecord);
+                } catch (FileNotFoundException | StorageException e) {
+                    throw new DataRetrievalException(e.getLocalizedMessage(),
+                            e);
+                }
+
+                DefaultGeometryData[] geomDatas = null;
+                // TODO: Update this when more radar graphics are supported
+                if (radarInfo.getProductCode() == 166) {
+                    geomDatas = RadarGeometryDataUtil
+                            .makeMeltingLayerGeom(radarRecord);
+                } else if (radarInfo.getProductCode() == 141) {
+                    geomDatas = RadarGeometryDataUtil
+                            .makeMesocycloneGeom(radarRecord);
+                } else if (radarInfo.getProductCode() == 58) {
+                    geomDatas = RadarGeometryDataUtil
+                            .makeStormTrackGeom(radarRecord);
+                } else if (radarInfo.getProductCode() == 59) {
+                    geomDatas = RadarGeometryDataUtil
+                            .makeHailIndexGeom(radarRecord);
+                } else if (radarInfo.getProductCode() == 61) {
+                    geomDatas = RadarGeometryDataUtil.makeTVSGeom(radarRecord);
+                } else {
+                    throw new DataRetrievalException(
+                            "Product code " + radarInfo.getProductCode()
+                                    + " is not yet supported.");
+                }
+                // reverse map parameter to match request.
+                List<String> requestedParameters = Arrays
+                        .asList(request.getParameters());
+                String productCode = radarRecord.getProductCode().toString();
+                for (DefaultGeometryData geomData : geomDatas) {
+                    if (requestedParameters.contains(radarInfo.getName())) {
+                        geomData.addData(radarInfo.getName(),
+                                radarInfo.getName());
+                    } else if (requestedParameters
+                            .contains(radarInfo.getMnemonic())) {
+                        geomData.addData(radarInfo.getMnemonic(),
+                                radarInfo.getMnemonic());
+                    } else {
+                        geomData.addData(productCode, productCode);
+                    }
+
+                    // set remaining metadata
+                    geomData.setDataTime(radarRecord.getDataTime());
+                    geomData.setLevel(getLevel(radarRecord, request));
+                    geomData.setLocationName(generateLocationName(radarRecord));
+                    geomData.addAttribute(ICAO, radarRecord.getIcao());
+                    geomData.addAttribute(FORMAT, radarRecord.getFormat());
+                    rval.add(geomData);
+                }
+            }
+        }
+        return rval.toArray(new IGeometryData[rval.size()]);
+    }
+
+    /**
+     * Unpack records from response and group by HDF5 file
+     *
+     * @param dbQueryResponse
+     * @return
+     */
+    private static Map<File, List<RadarRecord>> unpackResults(
+            DbQueryResponse dbQueryResponse) {
+        // Bin up requests to the same hdf5
+        Map<File, List<RadarRecord>> fileMap = new HashMap<>();
+
+        for (Map<String, Object> result : dbQueryResponse.getResults()) {
+            Object object = result.get(null);
+            if (object == null || !(object instanceof RadarRecord)) {
+                statusHandler.warn(
+                        "Unexpected DB query result for radar: " + object);
+                continue;
+            }
+            RadarRecord record = (RadarRecord) object;
+            File hdf5File = PDOUtil.getHDF5File(record);
+            List<RadarRecord> recList = fileMap.get(hdf5File);
+            if (recList == null) {
+                recList = new ArrayList<>();
+                fileMap.put(hdf5File, recList);
+            }
+            recList.add(record);
+        }
+        return fileMap;
+    }
+
+    /**
+     * Get the level for the radar record. If the request specifies
+     * {@value #LEVEL_ONE} or {@value #LEVEL_TWO} identifiers, those fields will
+     * be used. If {@value #LEVEL_ONE} is not specified, {@value #PRIMARY_ANGLE}
+     * will be used.
+     *
+     * @param radarRecord
+     *            The radar record.
+     * @param request
+     *            The request.
+     * @return The created level.
+     */
+    private Level getLevel(RadarRecord radarRecord, IDataRequest request) {
+        String levelOneField = getLevelField(request, LEVEL_ONE);
+        String levelTwoField = getLevelField(request, LEVEL_TWO);
+
+        if (levelOneField == null) {
+            levelOneField = PRIMARY_ANGLE;
+        }
+
+        Level level;
+        if (PRIMARY_ANGLE.equals(levelOneField)) {
+            level = getTiltLevel(radarRecord.getPrimaryElevationAngle());
+        } else if (TRUE_ANGLE.equals(levelOneField)) {
+            level = getTiltLevel(radarRecord.getTrueElevationAngle());
+        } else { // elevationNumber
+            level = new Level();
+            level.setMasterLevel(new MasterLevel("OSEQD"));
+            level.setLevelonevalue(radarRecord.getElevationNumber());
+        }
+
+        if (levelTwoField != null) {
+            if (PRIMARY_ANGLE.equals(levelTwoField)) {
+                level.setLeveltwovalue(radarRecord.getPrimaryElevationAngle());
+            } else if (TRUE_ANGLE.equals(levelTwoField)) {
+                level.setLeveltwovalue(radarRecord.getTrueElevationAngle());
+            } else { // elevationNumber
+                level.setLeveltwovalue(radarRecord.getElevationNumber());
+            }
+        }
+
+        return level;
+    }
+
+    /**
+     * Get the level field corresponding to the given key from the request's
+     * identifiers.
+     *
+     * @param request
+     * @param levelFieldKey
+     *            key indicating level one or level two
+     * @return the level field (may be null)
+     */
+    private String getLevelField(IDataRequest request, String levelFieldKey) {
+        Object levelField = request.getIdentifiers().get(levelFieldKey);
+        if (levelField == null) {
+            return null;
+        } else if (levelField instanceof String
+                && SUPPORTED_LEVELS.contains(levelField)) {
+            return (String) levelField;
+        }
+
+        // Nothing else is valid since the value is a DB field name
+        throw new IncompatibleRequestException(
+                "'" + levelField.toString() + "' is not a valid level field");
+    }
+
+    /**
+     * Get a unique name describing the location of the radar data. The name
+     * always includes icao, elevation angle, num bins and num radials. For
+     * radial data it also includes the first and last angle of the radial data.
+     * Theoretically two radial geometries could have the same name but
+     * internally different angleData but this is very unlikely and the points
+     * would be very nearly identical.
+     *
+     *
+     * @param radarRecord
+     *            a record.
+     * @return A unique location name
+     */
+    protected String generateLocationName(RadarRecord radarRecord) {
+        StringBuilder locationName = new StringBuilder(32);
+        locationName.append(radarRecord.getIcao());
+        locationName.append("_");
+        locationName.append(radarRecord.getTrueElevationAngle());
+        locationName.append("_");
+        locationName.append(radarRecord.getNumBins());
+        locationName.append("_");
+        locationName.append(radarRecord.getNumRadials());
+        float[] angleData = radarRecord.getAngleData();
+        if (angleData != null) {
+            locationName.append("_");
+            locationName.append(angleData[0]);
+            locationName.append("_");
+            locationName.append(angleData[angleData.length - 1]);
+        }
+        return locationName.toString();
+    }
+
+    protected RadarRecord asRadarRecord(PluginDataObject pdo) {
+        if (pdo instanceof RadarRecord) {
+            return (RadarRecord) pdo;
+        }
+        throw new DataRetrievalException(this.getClass().getSimpleName()
+                + " cannot handle " + pdo.getClass().getSimpleName());
+    }
+
+    @Override
+    protected GridGeometry2D getGridGeometry(PluginDataObject pdo) {
+        RadarRecord radarRecord = asRadarRecord(pdo);
+        if (radarRecord.getFormat().equals(RADIAL_FORMAT)) {
+            try {
+                float[] angleData = radarRecord.getAngleData();
+                if (angleData == null) {
+                    populateRecord(radarRecord);
+                    angleData = radarRecord.getAngleData();
+                }
+                if (angleData == null) {
+                    return null;
+                }
+
+                // NOTE: do not set swapXY=true even though it matches the raw
+                // data better because there is lots of code, especially on the
+                // Viz side that does not correctly handle the resulting
+                // GridGeometry.
+                return RadarProjectionFactory.constructGridGeometry(
+                        new Coordinate(radarRecord.getLongitude(),
+                                radarRecord.getLatitude()),
+                        angleData, radarRecord.getGateResolution(),
+                        radarRecord.getTrueElevationAngle(),
+                        radarRecord.getNumBins(), false);
+            } catch (FactoryException e) {
+                throw new DataRetrievalException(e);
+            }
+        } else if (radarRecord.getFormat().equals(RASTER_FORMAT)) {
+            double maxExtent = RadarUtil.calculateExtent(radarRecord);
+            return RadarUtil.constructGridGeometry(radarRecord.getCRS(),
+                    maxExtent, Math.max(radarRecord.getNumBins(),
+                            radarRecord.getNumRadials()));
+
+        } else {
+            return super.getGridGeometry(pdo);
+        }
+    }
+
+    @Override
+    protected SubGridGeometryCalculator calculateSubGrid(
+            ReferencedEnvelope envelope, GridGeometry2D gridGeometry)
+            throws EnvelopeProjectionException {
+        /*
+         * The SubGridGeometryCalculator cannot accurately calculate subgrids
+         * into RadialBin projections. For this factory the request envelope is
+         * only used to limit the sites, not to subgrid. Returning null causes
+         * the super class to request a full grid.
+         */
+        return null;
+    }
+
+    @Override
+    protected DataSource getDataSource(PluginDataObject pdo,
+            SubGridGeometryCalculator subGrid) {
+        RadarRecord radarRecord = asRadarRecord(pdo);
+        DataSource dataSource = getDataSource(radarRecord);
+        if (dataSource == null) {
+            /*
+             * Radial data prepopulates the record to get the gridGeometry but
+             * raster data waits until now.
+             */
+            populateRecord(radarRecord);
+            dataSource = getDataSource(radarRecord);
+            if (dataSource == null) {
+                throw new DataRetrievalException(
+                        "No grid data found for " + radarRecord);
+            }
+        }
+        if (radarRecord.getFormat().equals(RADIAL_FORMAT)) {
+            /*
+             * The raw data is in bin,radial format but the grid geometries we
+             * use are radial,bin so need to do some swapping.
+             */
+            dataSource = new AxisSwapDataSource(dataSource,
+                    radarRecord.getNumBins());
+        }
+
+        return dataSource;
+    }
+
+    /**
+     * Populate a DataSource from the raw data(byte or short) in the provided
+     * record.
+     *
+     * @param radarRecord
+     * @return a DataSource or null if the record is not populated or has no
+     *         grid data.
+     */
+    private DataSource getDataSource(RadarRecord radarRecord) {
+        int nx = radarRecord.getNumBins();
+        int ny = radarRecord.getNumRadials();
+        byte[] bytes = radarRecord.getRawData();
+        if (bytes != null) {
+            ByteBufferWrapper wrapper = new ByteBufferWrapper(bytes, nx, ny);
+            return FillValueFilter.apply((DataSource) wrapper, 0);
+        }
+        short[] shorts = radarRecord.getRawShortData();
+        if (shorts != null) {
+            ShortBufferWrapper wrapper = new ShortBufferWrapper(shorts, nx, ny);
+            return FillValueFilter.apply((DataSource) wrapper, 0);
+        }
+        return null;
+
+    }
+
+    protected void populateRecord(RadarRecord radarRecord)
+            throws DataRetrievalException {
+        try {
+            RadarDataRetriever.populateRadarRecord(
+                    PDOUtil.getDataStore(radarRecord), radarRecord);
+        } catch (Exception e) {
+            throw new DataRetrievalException(e);
+        }
+    }
+
+    @Override
+    protected Map<String, RequestConstraint> buildConstraintsFromRequest(
+            IDataRequest request) {
+        Map<String, RequestConstraint> constraints = new HashMap<>();
+        if (request.getParameters() != null
+                && request.getParameters().length > 0) {
+            Set<Integer> codes = new HashSet<>();
+            for (String parameter : request.getParameters()) {
+                codes.addAll(getProductCodesFromParameter(parameter));
+            }
+            RequestConstraint pcConstraint = new RequestConstraint(null,
+                    ConstraintType.IN);
+            for (Integer code : codes) {
+                pcConstraint.addToConstraintValueList(code.toString());
+            }
+            constraints.put(PRODUCT_CODE, pcConstraint);
+        }
+        Level[] levels = request.getLevels();
+        if (levels != null && levels.length > 0) {
+            RequestConstraint angleConstraint = new RequestConstraint(null,
+                    ConstraintType.IN);
+            RequestConstraint levelTwoConstraint = new RequestConstraint(null,
+                    ConstraintType.IN);
+
+            String levelOneField = getLevelField(request, LEVEL_ONE);
+            String levelTwoField = getLevelField(request, LEVEL_TWO);
+
+            if (levelOneField == null) {
+                levelOneField = PRIMARY_ANGLE;
+            }
+            for (Level level : levels) {
+                angleConstraint.addToConstraintValueList(
+                        level.getLevelOneValueAsString());
+                if (levelTwoField != null
+                        && level.getLeveltwovalue() != Level.INVALID_VALUE) {
+                    levelTwoConstraint.addToConstraintValueList(
+                            level.getLevelTwoValueAsString());
+                }
+            }
+            constraints.put(levelOneField, angleConstraint);
+            if (levelTwoConstraint.getConstraintValue() != null) {
+                constraints.put(levelTwoField, levelTwoConstraint);
+            }
+        }
+
+        String[] locations = request.getLocationNames();
+        if (locations != null && locations.length > 0) {
+            RequestConstraint rc = new RequestConstraint(locations);
+            constraints.put(ICAO, rc);
+        }
+
+        if (request.getEnvelope() != null) {
+            Envelope envelope = request.getEnvelope();
+
+            String minLon = Double.toString(envelope.getMinX());
+            String maxLon = Double.toString(envelope.getMaxX());
+            constraints.put(LONGITUDE, new RequestConstraint(minLon, maxLon));
+            String minLat = Double.toString(envelope.getMinY());
+            String maxLat = Double.toString(envelope.getMaxY());
+            constraints.put(LATITUDE, new RequestConstraint(minLat, maxLat));
+        }
+
+        Map<String, Object> identifiers = request.getIdentifiers();
+        if (identifiers != null && identifiers.containsKey(ICAO)) {
+            Object value = identifiers.get(ICAO);
+            if (value instanceof RequestConstraint) {
+                constraints.put(ICAO, (RequestConstraint) value);
+            } else {
+                constraints.put(ICAO, new RequestConstraint(value.toString()));
+            }
+        }
+
+        return constraints;
+    }
+
+    private Set<Integer> getProductCodesFromParameter(String parameter) {
+        String exception = null;
+        Set<Integer> codes = new HashSet<>();
+        for (RadarInfo info : getRadarInfo()) {
+            if (parameter.equals(info.getName())
+                    || parameter.equals(info.getMnemonic()) || parameter
+                            .equals(Integer.toString(info.getProductCode()))) {
+
+                if (SUPPORTED_FORMATS.contains(info.getFormat())) {
+                    codes.add(info.getProductCode());
+                } else {
+                    exception = info.getFormat() + " is not supported";
+                }
+            }
+        }
+        if (codes.isEmpty()) {
+            // If any valid product codes are found then don't complain.
+            if (exception != null) {
+                throw new DataRetrievalException(exception);
+            } else {
+                throw new DataRetrievalException(
+                        parameter + " is not a valid radar parameter.");
+            }
+        }
+        return codes;
+    }
+
+    private static synchronized Level getTiltLevel(double angle) {
+        if (tiltLevel == null) {
+            tiltLevel = new MasterLevel("TILT");
+            tiltLevel.setUnitString("°");
+            tiltLevel.setType("INC");
+            tiltLevel.setDescription("Tilt angle of a radar scan.");
+        }
+        Level level = new Level();
+        level.setMasterLevel(tiltLevel);
+        level.setLevelonevalue(angle);
+        return level;
+    }
+
+    private static synchronized RadarInfoDict getRadarInfo() {
+        if (radarInfo == null) {
+            File file = PathManagerFactory.getPathManager()
+                    .getStaticFile("radarInfo.txt");
+            if (file != null) {
+                radarInfo = RadarInfoDict.getInstance(file.getParent());
+            }
+        }
+        return radarInfo;
+    }
+
+    @Override
+    public String[] getAvailableLocationNames(IDataRequest request) {
+        return getAvailableValues(request, ICAO, String.class);
+    }
+
+    /**
+     * Get the available parameters for {@link #SUPPORTED_FORMATS supported
+     * formats}.
+     */
+    @Override
+    public String[] getAvailableParameters(IDataRequest request) {
+        DbQueryRequest dbQueryRequest = buildDbQueryRequest(request);
+        dbQueryRequest.addConstraint(FORMAT,
+                new RequestConstraint(SUPPORTED_FORMATS));
+        dbQueryRequest.setDistinct(Boolean.TRUE);
+        dbQueryRequest.addRequestField(PRODUCT_CODE);
+
+        DbQueryResponse dbQueryResponse = this
+                .executeDbQueryRequest(dbQueryRequest, request.toString());
+        Set<Integer> productCodes = new TreeSet<>();
+        for (Map<String, Object> result : dbQueryResponse.getResults()) {
+            productCodes.add((Integer) result.get(PRODUCT_CODE));
+        }
+        Set<String> parameters = new HashSet<>();
+        for (RadarInfo info : getRadarInfo()) {
+            if (productCodes.contains(Integer.valueOf(info.getProductCode()))) {
+                parameters.add(info.getName());
+                parameters.add(info.getMnemonic());
+                parameters.add(Integer.toString(info.getProductCode()));
+            }
+        }
+
+        return parameters.toArray(new String[0]);
+    }
+
+    /**
+     * Get the available levels. The optional identifiers, {@value #LEVEL_ONE}
+     * and {@value #LEVEL_TWO} can be supplied to choose which level fields are
+     * returned, otherwise {@value #PRIMARY_ANGLE} will be returned as the level
+     * one value.
+     */
+    @Override
+    public Level[] getAvailableLevels(IDataRequest request) {
+        DbQueryRequest dbQueryRequest = buildDbQueryRequest(request);
+        dbQueryRequest.setDistinct(Boolean.TRUE);
+
+        String levelOneField = getLevelField(request, LEVEL_ONE);
+        String levelTwoField = getLevelField(request, LEVEL_TWO);
+
+        if (levelOneField == null) {
+            levelOneField = PRIMARY_ANGLE;
+        }
+        dbQueryRequest.addRequestField(levelOneField);
+        if (levelTwoField != null) {
+            dbQueryRequest.addRequestField(levelTwoField);
+        }
+
+        DbQueryResponse dbQueryResponse = this
+                .executeDbQueryRequest(dbQueryRequest, request.toString());
+        Level level;
+        List<Level> levels = new ArrayList<>();
+        for (Map<String, Object> result : dbQueryResponse.getResults()) {
+            if (PRIMARY_ANGLE.equals(levelOneField)
+                    || TRUE_ANGLE.equals(levelTwoField)) {
+                level = getTiltLevel(
+                        Double.valueOf(result.get(levelOneField).toString()));
+            } else {
+                level = new Level();
+                level.setMasterLevel(new MasterLevel("OSEQD"));
+                level.setLevelonevalue(
+                        Double.valueOf(result.get(levelOneField).toString()));
+            }
+            if (levelTwoField != null) {
+                level.setLeveltwovalue(
+                        Double.valueOf(result.get(levelTwoField).toString()));
+            }
+            levels.add(level);
+        }
+
+        return levels.toArray(new Level[0]);
+    }
+
+    @Override
+    public String[] getOptionalIdentifiers(IDataRequest request) {
+        return new String[] { ICAO, LEVEL_ONE, LEVEL_TWO };
+    }
+
+    @Override
+    protected DbQueryRequest buildDbQueryRequest(IDataRequest request) {
+        validateLevelIdentifiers(request);
+        return super.buildDbQueryRequest(request);
+    }
+
+    /**
+     * Validates that, if specified, the {@value #LEVEL_ONE} and
+     * {@value #LEVEL_TWO} identifier values are supported.
+     *
+     * @param request
+     */
+    private void validateLevelIdentifiers(IDataRequest request) {
+        String levelOneField = getLevelField(request, LEVEL_ONE);
+        String levelTwoField = getLevelField(request, LEVEL_TWO);
+
+        if (levelOneField != null
+                && !SUPPORTED_LEVELS.contains(levelOneField)) {
+            throw new DataRetrievalException(LEVEL_ONE + LEVEL_ERROR);
+        }
+
+        if (levelTwoField != null
+                && !SUPPORTED_LEVELS.contains(levelTwoField)) {
+            throw new DataRetrievalException(LEVEL_TWO + LEVEL_ERROR);
+        }
+    }
+
+    /**
+     * Get allowed values for a specified identifier
+     *
+     * @param request
+     *            Request including the name of the datatype
+     * @param identifierKey
+     *            The identifier to get allowed values for
+     */
+    @Override
+    public String[] getIdentifierValues(IDataRequest request,
+            String identifierKey) {
+        if (!Arrays.asList(getRequiredIdentifiers(request))
+                .contains(identifierKey)
+                && !Arrays.asList(getOptionalIdentifiers(request))
+                        .contains(identifierKey)) {
+            throw new InvalidIdentifiersException(request.getDatatype(), null,
+                    Arrays.asList(new String[] { identifierKey }));
+        }
+        List<String> idValStrings;
+        if (identifierKey.equals(LEVEL_ONE)
+                || identifierKey.equals(LEVEL_TWO)) {
+            idValStrings = SUPPORTED_LEVELS;
+        } else {
+            Object[] idValues = getAvailableValues(request, identifierKey,
+                    Object.class);
+            idValStrings = new ArrayList<>(idValues.length);
+            for (Object idValue : idValues) {
+                idValStrings.add(idValue.toString());
+            }
+        }
+        return idValStrings.toArray(new String[idValStrings.size()]);
+    }
+
+    /**
+     *
+     * This is used to convert data from bin,radial format to radial bin format.
+     *
+     * <pre>
+     *
+     * SOFTWARE HISTORY
+     *
+     * Date         Ticket#    Engineer    Description
+     * ------------ ---------- ----------- --------------------------
+     * Jan 25, 2013            bsteffen     Initial creation
+     * Feb 14, 2013 1614       bsteffen    refactor data access framework to use
+     *                                          single request.
+     *
+     * </pre>
+     *
+     * @author bsteffen
+     * @version 1.0
+     */
+    private static class AxisSwapDataSource implements DataSource {
+
+        private final DataSource realData;
+
+        private final int numBins;
+
+        public AxisSwapDataSource(DataSource realData, int numBins) {
+            this.realData = realData;
+            this.numBins = numBins;
+        }
+
+        @Override
+        public double getDataValue(int x, int y) {
+            return realData.getDataValue(numBins - 1 - y, x);
+        }
+
+    }
+}

edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarGeometryDataUtil.java

@@ -0,0 +1,563 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ *
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ *
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ *
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.dataplugin.radar.dataaccess;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.geotools.coverage.grid.GeneralGridGeometry;
+import org.opengis.referencing.FactoryException;
+import org.opengis.referencing.operation.TransformException;
+
+import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData;
+import com.raytheon.uf.common.dataplugin.radar.RadarDataKey;
+import com.raytheon.uf.common.dataplugin.radar.RadarDataPoint;
+import com.raytheon.uf.common.dataplugin.radar.RadarRecord;
+import com.raytheon.uf.common.dataplugin.radar.level3.HdaHailPacket.HdaHailPoint;
+import com.raytheon.uf.common.dataplugin.radar.level3.SCITDataPacket;
+import com.raytheon.uf.common.dataplugin.radar.level3.SCITDataPacket.SCITDataCell;
+import com.raytheon.uf.common.dataplugin.radar.level3.SpecialGraphicSymbolPacket.SpecialGraphicPoint;
+import com.raytheon.uf.common.dataplugin.radar.level3.StormIDPacket;
+import com.raytheon.uf.common.dataplugin.radar.level3.StormIDPacket.StormIDPoint;
+import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket;
+import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPoint;
+import com.raytheon.uf.common.dataplugin.radar.level3.TVSPacket.TVSPoint;
+import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.MapValues;
+import com.raytheon.uf.common.dataplugin.radar.util.RadarRecordUtil;
+import com.raytheon.uf.common.geospatial.ReferencedCoordinate;
+import com.raytheon.uf.common.geospatial.ReferencedObject.Type;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.util.Pair;
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
+import com.vividsolutions.jts.geom.GeometryCollection;
+import com.vividsolutions.jts.geom.GeometryFactory;
+import com.vividsolutions.jts.geom.LinearRing;
+import com.vividsolutions.jts.geom.MultiPoint;
+import com.vividsolutions.jts.geom.Point;
+
+/**
+ * Methods that create IGeometryData for radar products
+ *
+ * <pre>
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 30, 2016 2671       tgurney     Initial creation
+ * Aug 31, 2016 2671       tgurney     Add makeMesocycloneGeom
+ * Sep 08, 2016 2671       tgurney     Add makeStormTrackGeom
+ * Sep 27, 2016 2671       tgurney     Add makeHailIndexGeom
+ * Sep 28, 2016 2671       tgurney     Add makeTVSGeom
+ *
+ * </pre>
+ *
+ * @author tgurney
+ */
+
+public class RadarGeometryDataUtil {
+
+    /**
+     * Data representing storm track for a single storm.
+     */
+    private static class StormTrackData {
+        /**
+         * Index of the point corresponding to the storm's current location. All
+         * points before that are past, and all points after that are forecast.
+         */
+        public Integer currentLocIndex;
+
+        public MultiPoint points;
+
+        public String stormId = "";
+    }
+
+    private static final GeometryFactory geomFactory = new GeometryFactory();
+
+    private static final transient IUFStatusHandler statusHandler = UFStatus
+            .getHandler(RadarGeometryDataUtil.class);
+
+    private RadarGeometryDataUtil() {
+        // static interface only
+    }
+
+    public static DefaultGeometryData[] makeMeltingLayerGeom(
+            RadarRecord radarRecord) {
+        Map<Integer, Coordinate[]> coords = RadarRecordUtil
+                .buildMeltingLayerCoordinates(radarRecord);
+        List<LinearRing> rings = new ArrayList<>();
+        for (Integer num : coords.keySet()) {
+            rings.add(geomFactory.createLinearRing(coords.get(num)));
+        }
+        GeometryCollection geomCollection = geomFactory
+                .createGeometryCollection(rings.toArray(new Geometry[0]));
+        DefaultGeometryData rval = new DefaultGeometryData();
+        rval.setGeometry(geomCollection);
+        return new DefaultGeometryData[] { rval };
+    }
+
+    /**
+     * @return (circleId, point) for specific location, or null if the data for
+     *         that location is nonexistent or invalid.
+     */
+    private static Pair<String, SpecialGraphicPoint> getMesocyclonePoint(
+            RadarDataKey currLoc, RadarRecord radarRecord) {
+        Integer productCode = radarRecord.getProductCode();
+        RadarDataPoint currStorm = radarRecord.getSymbologyData().get(currLoc);
+
+        HashMap<Integer, SymbologyPoint> displayPointData = currStorm
+                .getDisplayPointData().get(productCode);
+        if (displayPointData == null) {
+            statusHandler.warn("Mesocyclone at " + currLoc + " has no "
+                    + "points. Expected 1");
+            return null;
+        }
+
+        Collection<SymbologyPacket> textPackets = currStorm
+                .getDisplayPacketData().get(productCode).values();
+        if (textPackets.size() != 1) {
+            statusHandler.warn("Mesocyclone at " + currLoc + " has "
+                    + textPackets.size() + " text packets. Expected 1");
+            return null;
+        }
+
+        SymbologyPacket textPacket = textPackets.iterator().next();
+        if (!(textPacket instanceof TextSymbolPacket)) {
+            statusHandler.warn("Mesocyclone at " + currLoc + ": Expected "
+                    + "a " + TextSymbolPacket.class.getSimpleName() + ", got a "
+                    + textPacket.getClass().getName());
+            return null;
+        }
+
+        Collection<SymbologyPoint> currPoints = displayPointData.values();
+        if (currPoints.size() != 1) {
+            statusHandler.warn("Mesocyclone at " + currLoc + " has "
+                    + currPoints.size() + " points. Expected 1");
+            return null;
+        }
+
+        SymbologyPoint currPoint = currPoints.iterator().next();
+        if (!(currPoint instanceof SpecialGraphicPoint)) {
+            statusHandler.warn("Mesocyclone at " + currLoc + ": Expected "
+                    + "a " + SpecialGraphicPoint.class.getSimpleName()
+                    + ", got a " + currPoint.getClass().getName());
+            return null;
+        }
+
+        String circleId = ((TextSymbolPacket) textPacket).getTheText().trim();
+        SpecialGraphicPoint point = (SpecialGraphicPoint) currPoint;
+        return new Pair<>(circleId, point);
+    }
+
+    /**
+     * @return One geometry data per mesocyclone. The geometry itself is a
+     *         single point. Radius is stored in the "radiusKm" attribute. Other
+     *         tabular data are stored in the 'tableData' attribute.
+     */
+    public static DefaultGeometryData[] makeMesocycloneGeom(
+            RadarRecord radarRecord) {
+        List<DefaultGeometryData> geomDatas = new ArrayList<>();
+
+        for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) {
+            DefaultGeometryData geomData = new DefaultGeometryData();
+            Pair<String, SpecialGraphicPoint> mesoPoint = getMesocyclonePoint(
+                    currLoc, radarRecord);
+            if (mesoPoint == null) {
+                // failed validation
+                continue;
+            }
+            String circleId = mesoPoint.getFirst();
+            SpecialGraphicPoint graphicPoint = mesoPoint.getSecond();
+
+            // Make coordinate from graphic point
+            GeneralGridGeometry gg = RadarRecordUtil
+                    .getRadarGraphicsGridGeometry(radarRecord);
+            Coordinate coord = null;
+            try {
+                coord = new ReferencedCoordinate(
+                        RadarRecordUtil.rectifyCoordinate(
+                                new Coordinate(graphicPoint.i, graphicPoint.j)),
+                        gg, Type.GRID_CENTER).asLatLon();
+            } catch (TransformException | FactoryException e) {
+                statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
+                        e);
+                continue;
+            }
+
+            // Get tabular data
+            Map<MapValues, String> dataMap = radarRecord.getMapProductVals()
+                    .get(MapValues.MESO_TYPE).get(circleId);
+            if (dataMap == null) {
+                statusHandler.warn("Mesocyclone at " + currLoc
+                        + ": No tabular data found");
+                continue;
+            }
+            Map<String, String> newDataMap = new HashMap<>();
+            for (Entry<MapValues, String> mesoData : dataMap.entrySet()) {
+                // Remove MESO_ prefix
+                newDataMap.put(mesoData.getKey().toString().substring(5),
+                        mesoData.getValue());
+            }
+            // Radius is in 1/4 km; convert to km
+            double radius = graphicPoint.getPointFeatureAttr() / 4.0;
+            newDataMap.put("RADIUS_KM", Double.toString(radius));
+            geomData.addAttribute("tableData", newDataMap);
+            geomData.setGeometry(geomFactory.createPoint(coord));
+            geomDatas.add(geomData);
+        }
+        return geomDatas.toArray(new DefaultGeometryData[0]);
+    }
+
+    /**
+     * @return One geometry data per storm. The geometry itself is a multipoint.
+     *         Tabular data is stored in 'tableData' attribute. The current
+     *         location of the storm is the point with index specified by
+     *         "currentLocIndex" attribute. All points before that index are
+     *         past and all points after it are forecast.
+     */
+    public static DefaultGeometryData[] makeStormTrackGeom(
+            RadarRecord radarRecord) {
+        List<DefaultGeometryData> geomDatas = new ArrayList<>();
+
+        for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) {
+
+            DefaultGeometryData geomData = new DefaultGeometryData();
+            // Get storm ID and all points
+            RadarDataPoint currStorm = radarRecord.getSymbologyData()
+                    .get(currLoc);
+            StormTrackData stormTrackData = getStormTrackPoints(currStorm,
+                    RadarRecordUtil.getRadarGraphicsGridGeometry(radarRecord));
+            String stormId = stormTrackData.stormId;
+            if ("".equals(stormId)) {
+                statusHandler
+                        .warn("Storm track at " + currLoc + " has no storm ID");
+                continue;
+            }
+
+            // Get tabular data
+            Map<MapValues, String> dataMap = radarRecord.getMapProductVals()
+                    .get(MapValues.STI_TYPE).get(stormId);
+            if (dataMap == null) {
+                statusHandler.warn("Storm track " + stormId + " at " + currLoc
+                        + ": No tabular data found");
+                continue;
+            }
+            Map<String, String> newDataMap = new HashMap<>();
+            for (Entry<MapValues, String> stiData : dataMap.entrySet()) {
+                // Remove STI_ prefix
+                newDataMap.put(stiData.getKey().toString().substring(4),
+                        stiData.getValue());
+            }
+            geomData.addAttribute("currentLocIndex",
+                    stormTrackData.currentLocIndex.toString());
+            newDataMap.put("STORM_ID", stormId);
+            geomData.addAttribute("tableData", newDataMap);
+            geomData.setGeometry(stormTrackData.points);
+            geomDatas.add(geomData);
+        }
+
+        return geomDatas.toArray(new DefaultGeometryData[0]);
+    }
+
+    public static DefaultGeometryData[] makeHailIndexGeom(
+            RadarRecord radarRecord) {
+        List<DefaultGeometryData> geomDatas = new ArrayList<>();
+
+        for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) {
+
+            HashMap<Integer, HashMap<Integer, SymbologyPoint>> displayPointData = radarRecord
+                    .getSymbologyData().get(currLoc).getDisplayPointData();
+
+            HashMap<Integer, SymbologyPoint> pointMap = displayPointData
+                    .get(radarRecord.getProductCode());
+            if (pointMap == null) {
+                statusHandler.warn(
+                        "Hail index at " + currLoc + " has no " + "points.");
+                continue;
+            }
+
+            // Get coordinate and storm id
+            GeneralGridGeometry gg = RadarRecordUtil
+                    .getRadarGraphicsGridGeometry(radarRecord);
+            DefaultGeometryData geomData = new DefaultGeometryData();
+            String stormId = "";
+            Coordinate coord = null;
+            for (SymbologyPoint point : pointMap.values()) {
+                if (point instanceof HdaHailPoint) {
+                    try {
+                        coord = new ReferencedCoordinate(
+                                RadarRecordUtil.rectifyCoordinate(
+                                        new Coordinate(((HdaHailPoint) point).i,
+                                                ((HdaHailPoint) point).j)),
+                                gg, Type.GRID_CENTER).asLatLon();
+                    } catch (TransformException | FactoryException e) {
+                        statusHandler.handle(Priority.PROBLEM,
+                                e.getLocalizedMessage(), e);
+                        break;
+                    }
+                } else if (point instanceof StormIDPoint) {
+                    stormId = ((StormIDPoint) point).getStormID();
+                }
+            }
+            if (coord == null) {
+                continue;
+            }
+
+            // Get tabular data
+            Map<MapValues, String> dataMap = radarRecord.getMapProductVals()
+                    .get(MapValues.HAIL_TYPE).get(stormId);
+            if (dataMap == null) {
+                statusHandler.warn("Hail index " + stormId + " at " + currLoc
+                        + ": No tabular data found");
+                continue;
+            }
+            Map<String, String> newDataMap = new HashMap<>();
+            for (Entry<MapValues, String> stiData : dataMap.entrySet()) {
+                // Remove HI_ prefix
+                newDataMap.put(stiData.getKey().toString().substring(3),
+                        stiData.getValue());
+            }
+            newDataMap.put("STORM_ID", stormId);
+            geomData.addAttribute("tableData", newDataMap);
+            geomData.setGeometry(geomFactory.createPoint(coord));
+            geomDatas.add(geomData);
+        }
+
+        return geomDatas.toArray(new DefaultGeometryData[0]);
+    }
+
+    /**
+     * @return (stormId, point) for specific location, or null if the data for
+     *         that location is nonexistent or invalid.
+     */
+    private static Pair<String, TVSPoint> getTVSPoint(RadarDataKey currLoc,
+            RadarRecord radarRecord) {
+        Integer productCode = radarRecord.getProductCode();
+        RadarDataPoint currStorm = radarRecord.getSymbologyData().get(currLoc);
+
+        HashMap<Integer, SymbologyPoint> displayPointData = currStorm
+                .getDisplayPointData().get(productCode);
+        if (displayPointData == null) {
+            statusHandler.warn(
+                    "TVS at " + currLoc + " has no " + "points. Expected 1");
+            return null;
+        }
+
+        Collection<SymbologyPacket> displayPackets = currStorm
+                .getDisplayPacketData().get(productCode).values();
+        if (displayPackets.size() != 1) {
+            statusHandler.warn("TVS at " + currLoc + " has "
+                    + displayPackets.size() + " text packets. Expected 1");
+            return null;
+        }
+
+        SymbologyPacket stormIdPacket = displayPackets.iterator().next();
+        if (!(stormIdPacket instanceof StormIDPacket)) {
+            statusHandler.warn("TVS at " + currLoc + ": Expected " + "a "
+                    + StormIDPacket.class.getSimpleName() + ", got a "
+                    + stormIdPacket.getClass().getName());
+            return null;
+        }
+
+        Collection<SymbologyPoint> currPoints = displayPointData.values();
+        if (currPoints.size() != 1) {
+            statusHandler.warn("TVS at " + currLoc + " has " + currPoints.size()
+                    + " points. Expected 1");
+            return null;
+        }
+
+        SymbologyPoint currPoint = currPoints.iterator().next();
+        if (!(currPoint instanceof TVSPoint)) {
+            statusHandler.warn("TVS at " + currLoc + ": Expected " + "a "
+                    + TVSPoint.class.getSimpleName() + ", got a "
+                    + currPoint.getClass().getName());
+            return null;
+        }
+
+        StormIDPoint[] stormIdPoints = ((StormIDPacket) stormIdPacket)
+                .getPoints();
+        if (stormIdPoints.length != 1) {
+            statusHandler.warn("TVS at " + currLoc + " has "
+                    + stormIdPoints.length + " storm ID points. Expected 1");
+            return null;
+        }
+        TVSPoint point = (TVSPoint) currPoint;
+        String stormId = stormIdPoints[0].getStormID();
+        return new Pair<>(stormId, point);
+    }
+
+    public static DefaultGeometryData[] makeTVSGeom(RadarRecord radarRecord) {
+        List<DefaultGeometryData> geomDatas = new ArrayList<>();
+        GeneralGridGeometry gg = RadarRecordUtil
+                .getRadarGraphicsGridGeometry(radarRecord);
+
+        for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) {
+
+            DefaultGeometryData geomData = new DefaultGeometryData();
+            // Get point and storm ID
+            Pair<String, TVSPoint> tvsData = getTVSPoint(currLoc, radarRecord);
+            if (tvsData == null) {
+                // failed validation
+                continue;
+            }
+            String stormId = tvsData.getFirst();
+            TVSPoint point = tvsData.getSecond();
+            Coordinate coord;
+            try {
+                coord = new ReferencedCoordinate(
+                        RadarRecordUtil.rectifyCoordinate(
+                                new Coordinate(point.i, point.j)),
+                        gg, Type.GRID_CENTER).asLatLon();
+            } catch (TransformException | FactoryException e) {
+                statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
+                        e);
+                continue;
+            }
+
+            // Get tabular data
+            Map<MapValues, String> dataMap = radarRecord.getMapProductVals()
+                    .get(MapValues.TVS_TYPE).values().iterator().next();
+            if (dataMap == null) {
+                statusHandler.warn("TVS " + stormId + " at " + currLoc
+                        + ": No tabular data found");
+                continue;
+            }
+            Map<String, String> newDataMap = new HashMap<>();
+            for (Entry<MapValues, String> stiData : dataMap.entrySet()) {
+                // Remove TVS_ prefix
+                newDataMap.put(stiData.getKey().toString().substring(4),
+                        stiData.getValue());
+            }
+            newDataMap.put("STORM_ID", stormId);
+            newDataMap.put("ELEVATED", point.elevated ? "Y" : "N");
+
+            geomData.addAttribute("tableData", newDataMap);
+            geomData.setGeometry(geomFactory.createPoint(coord));
+            geomDatas.add(geomData);
+        }
+        return geomDatas.toArray(new DefaultGeometryData[0]);
+    }
+
+    private static StormTrackData getStormTrackPoints(RadarDataPoint stiPoint,
+            GeneralGridGeometry gridGeometry) {
+        List<Point> points = new ArrayList<>();
+        SCITDataPacket forecastPacket = null;
+        SCITDataPacket pastPacket = null;
+        TextSymbolPacket currentPacket = null;
+        StormIDPoint stormId = null;
+        StormTrackData stormTrackData = new StormTrackData();
+
+        HashMap<Integer, HashMap<Integer, SymbologyPacket>> displayPacketData = stiPoint
+                .getDisplayPacketData();
+
+        for (Integer type : displayPacketData.keySet()) {
+            for (SymbologyPacket packet : displayPacketData.get(type)
+                    .values()) {
+                if (packet instanceof SCITDataPacket) {
+                    for (SCITDataCell currCell : ((SCITDataPacket) packet)
+                            .getPoints()) {
+                        if (currCell.getText().contains("!")) {
+                            // '!' indicates past
+                            pastPacket = (SCITDataPacket) packet;
+                            continue;
+                        } else if (currCell.getText().contains("#")) {
+                            // '#' indicates forecast
+                            forecastPacket = (SCITDataPacket) packet;
+                            continue;
+                        }
+                    }
+                } else if (packet instanceof TextSymbolPacket) {
+                    if (((TextSymbolPacket) packet).getTheText()
+                            .contains("\"")) {
+                        // '"' indicates current storm location
+                        currentPacket = (TextSymbolPacket) packet;
+                    }
+                } else if (packet instanceof StormIDPacket) {
+                    // Should only have one point
+                    stormId = ((StormIDPacket) packet).getPoints()[0];
+                }
+            }
+        }
+
+        if (pastPacket != null) {
+            points.addAll(createSCITDataCell(pastPacket, gridGeometry));
+        }
+        if (currentPacket != null) {
+            points.addAll(createSCITDataCell(currentPacket, gridGeometry));
+            stormTrackData.currentLocIndex = points.size() - 1;
+        }
+        if (forecastPacket != null) {
+            points.addAll(createSCITDataCell(forecastPacket, gridGeometry));
+        }
+        stormTrackData.points = geomFactory
+                .createMultiPoint(points.toArray(new Point[0]));
+        if (stormId != null) {
+            stormTrackData.stormId = stormId.getStormID();
+        }
+        return stormTrackData;
+    }
+
+    /**
+     * @return List of points from a single SymbologyPacket
+     */
+    private static List<Point> createSCITDataCell(SymbologyPacket packet,
+            GeneralGridGeometry gridGeometry) {
+        List<Point> points = new ArrayList<>();
+        List<SCITDataCell> cells;
+        if (packet instanceof TextSymbolPacket) {
+            TextSymbolPacket pkt = (TextSymbolPacket) packet;
+            cells = new ArrayList<>();
+            cells.add(new SCITDataCell());
+            cells.get(0).setText(pkt.getTheText());
+            cells.get(0).setI(pkt.getI());
+            cells.get(0).setJ(pkt.getJ());
+        } else {
+            SCITDataPacket pkt = (SCITDataPacket) packet;
+            cells = pkt.getPoints();
+        }
+        if (cells == null) {
+            return points;
+        }
+
+        for (SCITDataCell point : cells) {
+            Coordinate coord = null;
+            try {
+                coord = new ReferencedCoordinate(
+                        RadarRecordUtil.rectifyCoordinate(
+                                new Coordinate(point.i, point.j)),
+                        gridGeometry, Type.GRID_CENTER).asLatLon();
+            } catch (TransformException | FactoryException e) {
+                statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(),
+                        e);
+                continue;
+            }
+            points.add(geomFactory.createPoint(coord));
+        }
+        return points;
+    }
+}

edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/level3/SCLPacket.java

@@ -0,0 +1,55 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.dataplugin.radar.level3;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
+
+/**
+ * This is a SCL centric version of the Generic Packet
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 18, 2016 DR 18796   jdynina    Initial creation
+ * Mar 06, 2016 DR 19848   jdynina    Renamed SCC to SCL
+ * 
+ * </pre>
+ * 
+ * @author jdynina
+ * @version 1.0
+ */
+
+@DynamicSerialize
+public class SCLPacket extends GenericDataPacket {
+
+    public SCLPacket(int packetId, DataInputStream in) throws IOException {
+        super(packetId, in);
+    }
+
+    public SCLPacket() {
+
+    }
+
+}

edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java

@@ -27,7 +27,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.regex.Matcher;
 
+import org.geotools.coverage.grid.GeneralGridEnvelope;
+import org.geotools.coverage.grid.GeneralGridGeometry;
+import org.geotools.geometry.GeneralEnvelope;
 import org.geotools.referencing.GeodeticCalculator;
+import org.opengis.referencing.FactoryException;
+import org.opengis.referencing.operation.TransformException;
 
 import com.raytheon.uf.common.dataplugin.radar.RadarDataKey;
 import com.raytheon.uf.common.dataplugin.radar.RadarDataPoint;
@@ -36,6 +41,8 @@ import com.raytheon.uf.common.dataplugin.radar.level3.DMDPacket;
 import com.raytheon.uf.common.dataplugin.radar.level3.DMDPacket.DMDAttributeIDs;
 import com.raytheon.uf.common.dataplugin.radar.level3.GraphicBlock;
 import com.raytheon.uf.common.dataplugin.radar.level3.Layer;
+import com.raytheon.uf.common.dataplugin.radar.level3.LinkedContourVectorPacket;
+import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVector;
 import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyBlock;
 import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket;
 import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket;
@@ -44,6 +51,11 @@ import com.raytheon.uf.common.dataplugin.radar.level3.generic.AreaComponent.Area
 import com.raytheon.uf.common.dataplugin.radar.level3.generic.GenericDataComponent;
 import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.DHRValues;
 import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.GraphicBlockValues;
+import com.raytheon.uf.common.geospatial.ReferencedCoordinate;
+import com.raytheon.uf.common.geospatial.ReferencedObject.Type;
+import com.raytheon.uf.common.status.IUFStatusHandler;
+import com.raytheon.uf.common.status.UFStatus;
+import com.raytheon.uf.common.status.UFStatus.Priority;
 import com.vividsolutions.jts.geom.Coordinate;
 
 /**
@@ -60,45 +72,53 @@ import com.vividsolutions.jts.geom.Coordinate;
  * Mar 19, 2013 1804       bsteffen    Remove empty data structures from radar
  *                                     hdf5.
  * Sep 03, 2013 DR 13083   gzhang      Add DHR Bias support for ADAD(38)/(46).
+ * Aug 26, 2016 2671       tgurney     Extract buildMeltingLayerCoordinates from
+ *                                     RadarMLResource
+ * Aug 31, 2016 2671       tgurney     Make rectifyCoordinate public. Extract
+ *                                     new method getRadarGridGeometry from
+ *                                     buildMeltingLayerCoordinates
  * </pre>
  *
  * @author mnash
- * @version 1.0
  */
 
 public class RadarRecordUtil {
     private static final String NO_STORMS_DETECTED = "NO STORMS DETECTED";
 
+    private static final transient IUFStatusHandler statusHandler = UFStatus
+            .getHandler(RadarRecordUtil.class);
+
     public static Map<String, Map<GraphicBlockValues, String>> parseGraphicBlock(
             RadarRecord record) {
 
-        Map<String, Map<GraphicBlockValues, String>> values = new LinkedHashMap<String, Map<GraphicBlockValues, String>>();
+        Map<String, Map<GraphicBlockValues, String>> values = new LinkedHashMap<>();
 
         if (record != null) {
             GraphicBlock gb = record.getGraphicBlock();
             if (gb != null) {
                 Layer[] pages = gb.getPages();
-                for (int i = 0; i < pages.length; i++) {
+                for (Layer page : pages) {
-                    SymbologyPacket[] packets = pages[i].getPackets();
+                    SymbologyPacket[] packets = page.getPackets();
                     for (int j = 1; j < packets.length; j++) {
                         if (packets[j] instanceof TextSymbolPacket) {
                             if (!NO_STORMS_DETECTED
                                     .equals(((TextSymbolPacket) packets[j])
                                             .getTheText())) {
-                                Map<GraphicBlockValues, String> map = new HashMap<GraphicBlockValues, String>();
+                                Map<GraphicBlockValues, String> map = new HashMap<>();
                                 Matcher m = RadarConstants.graphic_block_pattern
-                                        .matcher(getNormalizedGBText(((TextSymbolPacket) packets[j])
+                                        .matcher(getNormalizedGBText(
+                                                ((TextSymbolPacket) packets[j])
                                                         .getTheText()));
                                 if (m.find()) {
                                     String storm_id = m.group(1).trim();
-                                    map.put(GraphicBlockValues.AZIMUTH, m
+                                    map.put(GraphicBlockValues.AZIMUTH,
-                                            .group(2).trim());
+                                            m.group(2).trim());
-                                    map.put(GraphicBlockValues.RANGE, m
+                                    map.put(GraphicBlockValues.RANGE,
-                                            .group(3).trim());
+                                            m.group(3).trim());
-                                    map.put(GraphicBlockValues.TVS, m.group(4)
+                                    map.put(GraphicBlockValues.TVS,
-                                            .trim());
+                                            m.group(4).trim());
-                                    map.put(GraphicBlockValues.MDA, m.group(5)
+                                    map.put(GraphicBlockValues.MDA,
-                                            .trim());
+                                            m.group(5).trim());
                                     String temp = m.group(6).trim();
                                     if ("UNKNOWN".equals(temp)) {
                                         map.put(GraphicBlockValues.POSH, temp);
@@ -113,14 +133,14 @@ public class RadarRecordUtil {
                                         map.put(GraphicBlockValues.MXHAILSIZE,
                                                 temp.split("/")[2].trim());
                                     }
-                                    map.put(GraphicBlockValues.VIL, m.group(7)
+                                    map.put(GraphicBlockValues.VIL,
-                                            .trim());
+                                            m.group(7).trim());
-                                    map.put(GraphicBlockValues.DBZM, m.group(8)
+                                    map.put(GraphicBlockValues.DBZM,
-                                            .trim());
+                                            m.group(8).trim());
-                                    map.put(GraphicBlockValues.HT, m.group(9)
+                                    map.put(GraphicBlockValues.HT,
-                                            .trim());
+                                            m.group(9).trim());
-                                    map.put(GraphicBlockValues.TOP, m.group(10)
+                                    map.put(GraphicBlockValues.TOP,
-                                            .trim());
+                                            m.group(10).trim());
                                     temp = m.group(11).trim();
                                     if ("NEW".equals(temp)) {
                                         map.put(GraphicBlockValues.FCSTDIR,
@@ -152,7 +172,8 @@ public class RadarRecordUtil {
         for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) {
             RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon);
 
-            for (Integer type : currPoint.getDisplayGenericPointData().keySet()) {
+            for (Integer type : currPoint.getDisplayGenericPointData()
+                    .keySet()) {
                 for (GenericDataComponent currComponent : currPoint
                         .getDisplayGenericPointData().get(type).values()) {
                     if (featureId.equalsIgnoreCase(currComponent
@@ -181,7 +202,8 @@ public class RadarRecordUtil {
 
         for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) {
             RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon);
-            for (Integer type : currPoint.getDisplayGenericPointData().keySet()) {
+            for (Integer type : currPoint.getDisplayGenericPointData()
+                    .keySet()) {
                 for (GenericDataComponent currComponent : currPoint
                         .getDisplayGenericPointData().get(type).values()) {
                     if (featureId.equalsIgnoreCase(currComponent
@@ -195,14 +217,15 @@ public class RadarRecordUtil {
     }
 
     public static List<String> getDMDFeatureIDs(RadarRecord record) {
-        List<String> rval = new ArrayList<String>();
+        List<String> rval = new ArrayList<>();
         SymbologyBlock sb = record.getSymbologyBlock();
         if (sb != null) {
             for (Layer layer : sb.getLayers()) {
-                for (SymbologyPacket packet : layer.getPackets())
+                for (SymbologyPacket packet : layer.getPackets()) {
                     rval.addAll(((DMDPacket) packet).getFeatureIDs());
                 }
             }
+        }
         return rval;
     }
 
@@ -213,7 +236,8 @@ public class RadarRecordUtil {
         for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) {
             RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon);
 
-            for (Integer type : currPoint.getDisplayGenericPointData().keySet()) {
+            for (Integer type : currPoint.getDisplayGenericPointData()
+                    .keySet()) {
                 for (GenericDataComponent currComponent : currPoint
                         .getDisplayGenericPointData().get(type).values()) {
                     currFeature = (AreaComponent) currComponent;
@@ -240,11 +264,11 @@ public class RadarRecordUtil {
      */
     public List<String> getFeatures(RadarRecord record,
             RadarConstants.MapValues type, String id) {
-        List<String> list = new ArrayList<String>();
+        List<String> list = new ArrayList<>();
         for (Map.Entry<String, Map<RadarConstants.MapValues, String>> entry : record
                 .getMapProductVals().get(type).entrySet()) {
-            String fid = entry.getValue().get(
+            String fid = entry.getValue()
-                    RadarConstants.MapValues.MESO_STORM_ID);
+                    .get(RadarConstants.MapValues.MESO_STORM_ID);
             if (fid != null) {
                 fid = fid.trim();
                 if (id.trim().equals(fid)) {
@@ -285,8 +309,9 @@ public class RadarRecordUtil {
         GeodeticCalculator gd = new GeodeticCalculator();
         gd.setStartingGeographicPoint(lng, lat);
         gd.setDirection(calcDir, rng);
-        Coordinate coor = new Coordinate(gd.getDestinationGeographicPoint()
+        Coordinate coor = new Coordinate(
-                .getX(), gd.getDestinationGeographicPoint().getY());
+                gd.getDestinationGeographicPoint().getX(),
+                gd.getDestinationGeographicPoint().getY());
         return coor;
     }
 
@@ -317,14 +342,15 @@ public class RadarRecordUtil {
             int maxBin = 0;
 
             // Loop through each bin, in each radial
-            for (int currRadial = 0; currRadial < record.getNumRadials(); currRadial++) {
+            for (int currRadial = 0; currRadial < record
+                    .getNumRadials(); currRadial++) {
                 // If it is moving, find the Integer delta value
                 if (record.srmSpeed != 0) {
                     // Get the delta value for the current radial
                     delta = record.srmSpeed
-                            * Math.cos((Math.PI / 180)
+                            * Math.cos(Math.PI / 180
-                                    * (record.srmDirection - record
+                                    * (record.srmDirection
-                                            .getAngleData()[currRadial]))
+                                            - record.getAngleData()[currRadial]))
                             / 1.944;
 
                     if (!record.isExpandedMode()) {
@@ -356,7 +382,8 @@ public class RadarRecordUtil {
                             record.srmData[currBin] = (byte) 1;
                         } else if (radialData[currBin] != 0) {
                             // Add delta to the radialPixel
-                            record.srmData[currBin] = (byte) (radialData[currBin] + deltaInt);
+                            record.srmData[currBin] = (byte) (radialData[currBin]
+                                    + deltaInt);
                         }
                     }
                 }
@@ -408,13 +435,14 @@ public class RadarRecordUtil {
         RadarRecordUtil.calculateSRM8(record);
     }
 
-    public static byte getSRMDataValue(RadarRecord record, int radial, int bin) {
+    public static byte getSRMDataValue(RadarRecord record, int radial,
+            int bin) {
         if (record.srmData == null) {
             RadarRecordUtil.calculateSRM8(record);
         }
 
-        return (record.srmData != null) ? record.srmData[radial
+        return record.srmData != null
-                * record.getNumBins() + bin] : 0;
+                ? record.srmData[radial * record.getNumBins() + bin] : 0;
     }
 
     public static boolean hasSRM(RadarRecord record) {
@@ -440,7 +468,7 @@ public class RadarRecordUtil {
             DHRValues.BIASAPPLIEDFLAG };
 
     public static Map<DHRValues, Double> getDHRValues(RadarRecord record) {
-        Map<DHRValues, Double> map = new HashMap<DHRValues, Double>();
+        Map<DHRValues, Double> map = new HashMap<>();
         String text = null;
         SymbologyBlock sb = record.getSymbologyBlock();
         if (sb != null) {
@@ -469,8 +497,9 @@ public class RadarRecordUtil {
         Integer flagZeroHybrid = null;
 
         String[] v = new String[nv];
-        for (vi = 0; vi < nv; ++vi)
+        for (vi = 0; vi < nv; ++vi) {
             v[vi] = text.substring(vi * 8, (vi + 1) * 8);
+        }
 
         vi = 0;
         while (vi < nv) {
@@ -480,8 +509,9 @@ public class RadarRecordUtil {
                 map.put(DHRValues.PRECIPCAT, (double) precipCat);
                 vi += 6;
             } else if (s.equals("ADAP(32)")) {
-                for (int i = 0; i < ADAP32_VALUES.length; ++i)
+                for (DHRValues element : ADAP32_VALUES) {
-                    map.put(ADAP32_VALUES[i], parseDHRValue(v[vi++]));
+                    map.put(element, parseDHRValue(v[vi++]));
+                }
                 biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;
                 while (vi < nv) {
                     s = v[vi++];
@@ -493,8 +523,9 @@ public class RadarRecordUtil {
                          * parseDHRValue(text, vi + 1));
                          */
                         flagZeroHybrid = (int) parseDHRValue(v[vi + 2]);
-                        if (flagZeroHybrid != 0 && flagZeroHybrid != 1)
+                        if (flagZeroHybrid != 0 && flagZeroHybrid != 1) {
                             flagZeroHybrid = 0; // should print warning
+                        }
                         vi += 15;
                     } else if (s.equals("BIAS(11)")) {
                         biasCalculated = parseDHRValue(v[vi + 8]);
@@ -506,14 +537,17 @@ public class RadarRecordUtil {
                 // from A1 decodeDHR.C.
                 map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9]));
                 map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10]));
-                map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25]));
+                map.put(DHRValues.MAXPRECIPRATEALLOW,
-                map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[vi + 37]));	biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083
+                        parseDHRValue(v[vi + 25]));
+                map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[vi + 37]));
+                biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083
                 s = v[46];
                 if (s.equals("SUPL(15)")) {
                     biasCalculated = parseDHRValue(v[71]);
                     flagZeroHybrid = (int) parseDHRValue(v[49]);
-                    if (flagZeroHybrid != 0 && flagZeroHybrid != 1)
+                    if (flagZeroHybrid != 0 && flagZeroHybrid != 1) {
                         flagZeroHybrid = 0; // should print warning
+                    }
                 } else if (s.equals("SUPL(13)")) {
                     biasCalculated = parseDHRValue(v[69]);
                 }
@@ -521,23 +555,33 @@ public class RadarRecordUtil {
             } else if (s.equals("ADAP(46)")) {
                 map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9]));
                 map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10]));
-                map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25]));
+                map.put(DHRValues.MAXPRECIPRATEALLOW,
+                        parseDHRValue(v[vi + 25]));
                 s = v[68];
                 if (s.equals("BIAS(11)")) {
-                    map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53]));	biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083
+                    map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53]));
+                    biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR
+                                                                         // 13083
                     biasCalculated = parseDHRValue(v[77]);
                 } else if (s.equals("BIAS( 9)")) {
-                    map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53]));	biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083
+                    map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53]));
+                    biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR
+                                                                         // 13083
                     biasCalculated = parseDHRValue(v[73]);
                 }
                 vi = nv;
             }
         }
-        if (flagZeroHybrid != null)
+        if (flagZeroHybrid != null) {
             map.put(DHRValues.FLAGZEROHYBRID, (double) flagZeroHybrid);
+        }
         if (!biasApplied) {
             biasCalculated = 1.0;
-        } else { if(biasCalculated < 0.01 || biasCalculated > 100.0) biasCalculated = 1.0; } // DR 13083
+        } else {
+            if (biasCalculated < 0.01 || biasCalculated > 100.0) {
+                biasCalculated = 1.0;
+            }
+        } // DR 13083
         map.put(DHRValues.BIAS, biasCalculated);
 
         // Also include logic from A1 FFMPContainer::read(), FFMP_ORPG case
@@ -555,44 +599,49 @@ public class RadarRecordUtil {
 
     private static double parseDHRValue(String text) {
         String s = text.trim();
-        if (s.equals("T"))
+        if (s.equals("T")) {
             return 1;
-        else if (s.equals("F"))
+        } else if (s.equals("F")) {
             return 0;
-        else
+        } else {
             return Double.parseDouble(s);
         }
+    }
 
     /**
      * DR#11705: SCAN missing row(s) comparing to radar Comb Att Table.
      *
-     * Error cause: RadarConstants.GRAPHIC_BLOCK as a Regular Expression
+     * Error cause: RadarConstants.GRAPHIC_BLOCK as a Regular Expression pattern
-     * pattern can not match some variations in Graphic Block Texts with  
+     * can not match some variations in Graphic Block Texts with "<" and ">"
-     * "<" and ">" having spaces between them and their associated numbers
+     * having spaces between them and their associated numbers ( MXHAILSIZE and
-     * ( MXHAILSIZE and TOP ).
+     * TOP ).
      *
      * Fix: replace all "<" and ">" with space: " "
      *
-     * @param : Graphic Block Text that may contain ">" and/or "<".
+     * @param :
+     *            Graphic Block Text that may contain ">" and/or "<".
      * @return: String with ">" and/or "<" replaced by space.
      */
     private static String getNormalizedGBText(String text) {
 
         if (text == null || text.isEmpty()
-                || ((!text.contains(">")) && (!text.contains("<"))))
+                || !text.contains(">") && !text.contains("<")) {
             return text;
+        }
 
         /*
          * contains only ">"
          */
-        if (!text.contains("<"))
+        if (!text.contains("<")) {
             return text.replaceAll(">", " ");
+        }
 
         /*
          * contains only "<"
          */
-        if (!text.contains(">"))
+        if (!text.contains(">")) {
             return text.replaceAll("<", " ");
+        }
 
         /*
          * contains both "<" and ">"
@@ -600,4 +649,88 @@ public class RadarRecordUtil {
         return text.replaceAll(">", " ").replaceAll("<", " ");
 
     }
+
+    /**
+     * Add x and y offset needed to bring the radar coordinate system into
+     * something geotools can handle
+     */
+    public static Coordinate rectifyCoordinate(Coordinate c) {
+        c.x += 2048;
+        c.y += 2048;
+        c.y = 4096 - c.y;
+        return c;
+    }
+
+    /**
+     * Builds the necessary coordinates for use in the wireframe shapes
+     *
+     * @param radarRecord
+     * @return
+     */
+    public static Map<Integer, Coordinate[]> buildMeltingLayerCoordinates(
+            RadarRecord radarRecord) {
+        SymbologyBlock block = radarRecord.getSymbologyBlock();
+        GeneralGridGeometry gg = getRadarGraphicsGridGeometry(radarRecord);
+        ReferencedCoordinate coordinate;
+        Map<Integer, Coordinate[]> coordinates = new HashMap<>();
+        if (block != null) {
+            for (int i = 0; i < block.getNumLayers(); i++) {
+                for (int j = 1; j < block.getNumPackets(i); j++) {
+                    if (block.getPacket(i,
+                            j) instanceof LinkedContourVectorPacket) {
+                        List<LinkedVector> vector = ((LinkedContourVectorPacket) block
+                                .getPacket(i, j)).getVectors();
+                        Coordinate[] coords = new Coordinate[vector.size() + 1];
+                        for (int l = 0; l < coords.length - 1; l++) {
+                            if (!coordinates
+                                    .containsKey(vector.get(l).getTheColor())) {
+                                coordinates.put(
+                                        Integer.valueOf(
+                                                vector.get(l).getTheColor()),
+                                        coords);
+                            }
+
+                            // transform the coordinates to the correct
+                            // locations
+                            coordinate = new ReferencedCoordinate(
+                                    rectifyCoordinate(new Coordinate(
+                                            vector.get(l).getI2(),
+                                            vector.get(l).getJ2())),
+                                    gg, Type.GRID_CENTER);
+                            try {
+                                coords[l] = coordinate.asLatLon();
+                            } catch (TransformException | FactoryException e) {
+                                statusHandler.handle(Priority.PROBLEM,
+                                        e.getLocalizedMessage(), e);
+                            }
+                        }
+                        coords[coords.length - 1] = coords[0];
+                    }
+                }
+            }
+        }
+        return coordinates;
+    }
+
+    /**
+     * @param radarRecord
+     *            A radar graphics record
+     * @return Grid geometry suitable for radar graphics only. Not for other
+     *         types of radar products.
+     */
+    public static GeneralGridGeometry getRadarGraphicsGridGeometry(
+            RadarRecord radarRecord) {
+        GeneralEnvelope generalEnvelope = new GeneralEnvelope(2);
+        // Per section 3.3.3
+        generalEnvelope.setCoordinateReferenceSystem(radarRecord.getCRS());
+        generalEnvelope.setRange(0, -256000 * 2, 256000 * 2);
+        generalEnvelope.setRange(1, -256000 * 2, 256000 * 2);
+        // [-2048, 2048] == range of 4095 (inclusive 0), plus 1
+        // because GGR is exclusive (?)
+        GeneralGridGeometry rval = new GeneralGridGeometry(
+                new GeneralGridEnvelope(new int[] { 0, 0 },
+                        new int[] { 4096, 4096 }, false),
+                generalEnvelope);
+        return rval;
+    }
 }

edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarTextProductUtil.java

@@ -36,6 +36,7 @@ import java.util.List;
  *                                      dependency
  * Apr 14, 2016 DR18800    jdynina      Removed alerting
  * Apr 15, 2016 DR18796    jdynina      Added SCC
+ * Mar 03, 2017 DR19848    jdynina      Changed SCC to SCL to prevent conflicts
  * 
  * </pre>
  * 
@@ -80,7 +81,7 @@ public class RadarTextProductUtil {
             put(150, "WSRUSW");
             put(151, "WSRUSD");
             put(171, "WSRSTA");
-            put(202, "WSRSCC");
+            put(202, "WSRSCL");
         }
     };
 

edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/dataaccess/SatelliteGridFactory.java

@@ -68,6 +68,7 @@ import com.raytheon.uf.common.numeric.source.DataSource;
  *                                    IDataRequest
  * Jun 07, 2016  5574     tgurney     Add advanced query support
  * Aug 01, 2016  2416     tgurney     Add dataURI as optional identifier
+ * Mar 06, 2017  6142     bsteffen    Remove dataURI as optional identifier
  * 
  * </pre>
  * 
@@ -84,8 +85,7 @@ public class SatelliteGridFactory extends AbstractGridDataPluginFactory {
     private static final String FIELD_SOURCE = "source";
 
     private static final String[] OPTIONAL_IDENTIFIERS = { FIELD_SOURCE,
-            FIELD_CREATING_ENTITY, FIELD_SECTOR_ID, FIELD_PHYSICAL_ELEMENT,
+            FIELD_CREATING_ENTITY, FIELD_SECTOR_ID, FIELD_PHYSICAL_ELEMENT };
-            PluginDataObject.DATAURI_ID };
 
     public SatelliteGridFactory() {
         SatelliteUnits.register();
@@ -199,16 +199,17 @@ public class SatelliteGridFactory extends AbstractGridDataPluginFactory {
      */
     @Override
     public String[] getAvailableParameters(IDataRequest request) {
-        return getAvailableValues(request, FIELD_PHYSICAL_ELEMENT, String.class);
+        return getAvailableValues(request, FIELD_PHYSICAL_ELEMENT,
+                String.class);
     }
 
     @Override
     public String[] getIdentifierValues(IDataRequest request,
             String identifierKey) {
-        if (!Arrays.asList(getRequiredIdentifiers(request)).contains(
+        if (!Arrays.asList(getRequiredIdentifiers(request))
-                identifierKey)
+                .contains(identifierKey)
-                && !Arrays.asList(getOptionalIdentifiers(request)).contains(
+                && !Arrays.asList(getOptionalIdentifiers(request))
-                        identifierKey)) {
+                        .contains(identifierKey)) {
             throw new InvalidIdentifiersException(request.getDatatype(), null,
                     Arrays.asList(new String[] { identifierKey }));
         }

edexOsgi/com.raytheon.uf.edex.dataaccess/META-INF/MANIFEST.MF

@@ -2,7 +2,7 @@ Manifest-Version: 1.0
 Bundle-ManifestVersion: 2
 Bundle-Name: Data Access Framework EDEX
 Bundle-SymbolicName: com.raytheon.uf.edex.dataaccess
-Bundle-Version: 1.14.0.qualifier
+Bundle-Version: 1.16.0.qualifier
 Bundle-Vendor: RAYTHEON
 Require-Bundle: com.raytheon.uf.common.dataaccess,
  com.raytheon.uf.common.serialization.comm,

edexOsgi/com.raytheon.uf.edex.dataaccess/res/spring/dataaccess-request.xml

@@ -2,92 +2,29 @@
     xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
 
-
     <bean id="availableLocationsHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetAvailableLocationNamesHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetAvailableLocationNamesRequest" />
-        <constructor-arg ref="availableLocationsHandler" />
-    </bean>
-
     <bean id="availableTimesHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetAvailableTimesHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetAvailableTimesRequest" />
-        <constructor-arg ref="availableTimesHandler" />
-    </bean>
-
     <bean id="availableParametersHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetAvailableParametersHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetAvailableParametersRequest" />
-        <constructor-arg ref="availableParametersHandler" />
-    </bean>
-
     <bean id="availableLevelsHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetAvailableLevelsHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetAvailableLevelsRequest" />
-        <constructor-arg ref="availableLevelsHandler" />
-    </bean>
-
     <bean id="requiredIdentifiersHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetRequiredIdentifiersHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetRequiredIdentifiersRequest" />
-        <constructor-arg ref="requiredIdentifiersHandler" />
-    </bean>
-
     <bean id="optionalIdentifiersHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetOptionalIdentifiersHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetOptionalIdentifiersRequest" />
-        <constructor-arg ref="optionalIdentifiersHandler" />
-    </bean>
-
     <bean id="identifierValuesHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetIdentifierValuesHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetIdentifierValuesRequest" />
-        <constructor-arg ref="identifierValuesHandler" />
-    </bean>
-
     <bean id="supportedDatatypesHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetSupportedDatatypesHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetSupportedDatatypesRequest" />
-        <constructor-arg ref="supportedDatatypesHandler" />
-    </bean>
-
     <bean id="getGeometryDataHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetGeometryDataHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetGeometryDataRequest" />
-        <constructor-arg ref="getGeometryDataHandler" />
-    </bean>
-
     <bean id="getGridDataHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetGridDataHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
+    <bean id="getGridLatLonHandler"
-        <constructor-arg
+        class="com.raytheon.uf.edex.dataaccess.handler.GetGridLatLonHandler" />
-            value="com.raytheon.uf.common.dataaccess.request.GetGridDataRequest" />
-        <constructor-arg ref="getGridDataHandler" />
-    </bean>
-
     <bean id="getNotificationFilterHandler"
         class="com.raytheon.uf.edex.dataaccess.handler.GetNotificationFilterHandler" />
-    <bean factory-bean="handlerRegistry" factory-method="register">
+
-        <constructor-arg
-            value="com.raytheon.uf.common.dataaccess.request.GetNotificationFilterRequest" />
-        <constructor-arg ref="getNotificationFilterHandler" />
-    </bean>
 </beans>

edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridDataHandler.java

@@ -40,23 +40,15 @@ import com.raytheon.uf.common.util.CollectionUtil;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Jun 4, 2013             dgilling    Initial creation
+ * Oct 18, 2016 5916       bsteffen    Allow lazy loading of lat/lon data
  * 
  * </pre>
  * 
  * @author dgilling
- * @version 1.0
  */
-
 public final class GetGridDataHandler implements
         IRequestHandler<GetGridDataRequest> {
 
-    /*
-     * (non-Javadoc)
-     * 
-     * @see
-     * com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest
-     * (com.raytheon.uf.common.serialization.comm.IServerRequest)
-     */
     @Override
     public GetGridDataResponse handleRequest(final GetGridDataRequest request)
             throws Exception {
@@ -73,7 +65,7 @@ public final class GetGridDataHandler implements
         }
 
         GetGridDataResponse response = new GetGridDataResponse(
-                Arrays.asList(gridData));
+                Arrays.asList(gridData), request.isIncludeLatLonData());
         return response;
     }
 }

edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridLatLonHandler.java

@@ -0,0 +1,70 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.edex.dataaccess.handler;
+
+import org.geotools.coverage.grid.GridEnvelope2D;
+import org.geotools.coverage.grid.GridGeometry2D;
+import org.geotools.geometry.jts.ReferencedEnvelope;
+import org.geotools.referencing.CRS;
+import org.opengis.geometry.MismatchedDimensionException;
+import org.opengis.referencing.FactoryException;
+
+import com.raytheon.uf.common.dataaccess.request.GetGridLatLonRequest;
+import com.raytheon.uf.common.dataaccess.response.GetGridLatLonResponse;
+import com.raytheon.uf.common.geospatial.LatLonReprojection;
+import com.raytheon.uf.common.geospatial.LatLonWrapper;
+import com.raytheon.uf.common.serialization.comm.IRequestHandler;
+
+/**
+ * 
+ * Handler for {@link GetGridLatLonRequest}.
+ * 
+ * <pre>
+ *
+ * SOFTWARE HISTORY
+ * 
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- -----------------
+ * Oct 18, 2016  5916     bsteffen  Initial creation
+ * 
+ * </pre>
+ *
+ * @author bsteffen
+ */
+public final class GetGridLatLonHandler
+        implements IRequestHandler<GetGridLatLonRequest> {
+
+    @Override
+    public GetGridLatLonResponse handleRequest(GetGridLatLonRequest request)
+            throws MismatchedDimensionException, FactoryException {
+        GridEnvelope2D range = new GridEnvelope2D(0, 0, request.getNx(),
+                request.getNy());
+        ReferencedEnvelope envelope = new ReferencedEnvelope(
+                request.getEnvelope(), CRS.parseWKT(request.getCrsWkt()));
+        GridGeometry2D gridGeometry = new GridGeometry2D(range, envelope);
+        GetGridLatLonResponse response = new GetGridLatLonResponse();
+        LatLonWrapper latLonData = LatLonReprojection.getLatLons(gridGeometry);
+        response.setNx(request.getNx());
+        response.setNy(request.getNy());
+        response.setLats(latLonData.getLats());
+        response.setLons(latLonData.getLons());
+        return response;
+    }
+}

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py

@@ -18,7 +18,6 @@
 # further licensing information.
 ##
 
-
 #
 # Implements IData and wraps around a Java IData
 #  
@@ -32,6 +31,12 @@
 # 
 #
 
+##
+# This is a base file that is not intended to be overridden.
+##
+
+
+
 from awips.dataaccess import IData
 import JUtil, DataTime
 

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py

@@ -18,7 +18,6 @@
 # further licensing information.
 ##
 
-
 #
 # Implements IDataRequest and wraps around a Java IDataRequest
 #  
@@ -36,6 +35,12 @@
 # 
 #
 
+##
+# This is a base file that is not intended to be overridden.
+##
+
+
+
 from awips.dataaccess import IDataRequest
 from com.raytheon.uf.common.dataplugin.level import Level
 import JUtil

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py

@@ -18,7 +18,6 @@
 # further licensing information.
 ##
 
-
 #
 # Implements IGeometryData and wraps around a Java IGeometryData.
 #  
@@ -34,6 +33,12 @@
 #
 #
 
+##
+# This is a base file that is not intended to be overridden.
+##
+
+
+
 from awips.dataaccess import IGeometryData
 import JData
 

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py

@@ -18,7 +18,6 @@
 # further licensing information.
 # #
 
-
 #
 # Implements IGridData and wraps around a Java IGridData.
 #  
@@ -32,11 +31,16 @@
 #    08/06/14        3185          njensen        Only import Java classes when necessary
 #    Apr 23, 2015    4259          njensen        Updated for new JEP API
 #    11/02/15        5079          dgilling       Fix typo in getRawData.
-#    11/10/16        5900          bsteffen       Correct grid shape, return longitude
+#    10/14/16        5916          bsteffen       Correct grid shape, return longitude
-#    
 #
 #
 
+##
+# This is a base file that is not intended to be overridden.
+##
+
+
+
 from awips.dataaccess import IGridData
 import JData
 import numpy as np

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JepRouter.py

@@ -18,7 +18,6 @@
 # further licensing information.
 # #
 
-
 #
 # Routes requests to the Data Access Framework through JEP to the Java classes.
 # Returns Python objects that wrap Java objects.
@@ -43,7 +42,12 @@
 #                                                 getOptionalIdentifiers()
 #
 #
-#
+
+##
+# This is a base file that is not intended to be overridden.
+##
+
+
 
 from awips.dataaccess import IDataRequest
 

edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/__init__.py

@@ -18,7 +18,6 @@
 # further licensing information.
 ##
 
-
 #
 # __init__.py for dataaccess python that is used within a JVM.
 #  
@@ -30,7 +29,11 @@
 #    12/10/12                      njensen       Initial Creation.
 #
 #
-#
+
+##
+# This is a base file that is not intended to be overridden.
+##
+
 
 
 __all__ = [           

pythonPackages/dynamicserialize/ThriftSerializationContext.py

@@ -37,20 +37,27 @@
 #    06/12/13         #2099        dgilling      Implement readObject() and
 #                                                writeObject().
 #    Apr 24, 2015    4425          nabowle       Add Double support
+#    Oct 17, 2016    5919          njensen       Optimized for speed
 #
 #
 
 from thrift.Thrift import TType
-import inspect, sys, types
+import inspect
+import sys
+import types
+import time
 import dynamicserialize
 from dynamicserialize import dstypes, adapters
 import SelfDescribingBinaryProtocol
 import numpy
 
+DS_LEN = len('dynamicserialize.dstypes.')
+
 dsObjTypes = {}
 
+
 def buildObjMap(module):
-    if module.__dict__.has_key('__all__'):        
+    if '__all__' in module.__dict__:
         for i in module.__all__:
             name = module.__name__ + '.' + i
             __import__(name)
@@ -59,7 +66,7 @@ def buildObjMap(module):
         clzName = module.__name__[module.__name__.rfind('.') + 1:]
         clz = module.__dict__[clzName]
         tname = module.__name__
-        tname = tname.replace('dynamicserialize.dstypes.', '')        
+        tname = tname[DS_LEN:]
         dsObjTypes[tname] = clz
 
 buildObjMap(dstypes)
@@ -87,7 +94,9 @@ pythonToThriftMap = {
     numpy.int64: TType.I64
 }
 
-primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
+primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64,
+                    SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE)
+
 
 class ThriftSerializationContext(object):
 
@@ -141,7 +150,6 @@ class ThriftSerializationContext(object):
             TType.DOUBLE: self.protocol.writeF64List
         }
 
-    
     def readMessageStart(self):
         msg = self.protocol.readMessageBegin()
         return msg[0]
@@ -151,17 +159,19 @@ class ThriftSerializationContext(object):
 
     def deserializeMessage(self):
         name = self.protocol.readStructBegin()
-        name = name.replace('_', '.')
         if name.isdigit():
             obj = self._deserializeType(int(name))
             return obj
-        elif adapters.classAdapterRegistry.has_key(name):
+        name = name.replace('_', '.')
+        if name in adapters.classAdapterRegistry:
             return adapters.classAdapterRegistry[name].deserialize(self)
-        elif name.find('$') > -1:
+        elif '$' in name:
-            # it's an inner class, we're going to hope it's an enum, treat it special
+            # it's an inner class, we're going to hope it's an enum, treat it
+            # special
             fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
             if fieldName != '__enumValue__':
-                raise dynamiceserialize.SerializationException("Expected to find enum payload.  Found: " + fieldName)
+                raise dynamiceserialize.SerializationException(
+                    "Expected to find enum payload.  Found: " + fieldName)
             obj = self.protocol.readString()
             self.protocol.readFieldEnd()
             return obj
@@ -176,37 +186,30 @@ class ThriftSerializationContext(object):
         return obj
 
     def _deserializeType(self, b):
-        if self.typeDeserializationMethod.has_key(b):
+        try:
             return self.typeDeserializationMethod[b]()
-        else:
+        except KeyError:
-            raise dynamicserialize.SerializationException("Unsupported type value " + str(b))
+            raise dynamicserialize.SerializationException(
-    
+                "Unsupported type value " + str(b))
 
     def _deserializeField(self, structname, obj):
         fieldName, fieldType, fieldId = self.protocol.readFieldBegin()
         if fieldType == TType.STOP:
             return False
         elif fieldType != TType.VOID:
-#            if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName):
-#                result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self)
-#            else:
             result = self._deserializeType(fieldType)
             lookingFor = "set" + fieldName[0].upper() + fieldName[1:]
 
             try:
                 setMethod = getattr(obj, lookingFor)
-
-                if callable(setMethod):
                 setMethod(result)                
-                else:
-                    raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor)
             except:
-                raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor)
+                raise dynamicserialize.SerializationException(
+                    "Couldn't find setter method " + lookingFor)
 
         self.protocol.readFieldEnd()
         return True
 
-    
     def _deserializeArray(self):
         listType, size = self.protocol.readListBegin()
         result = []
@@ -241,19 +244,20 @@ class ThriftSerializationContext(object):
 
     def _lookupType(self, obj):
         pyt = type(obj)
-        if pythonToThriftMap.has_key(pyt):
+        if pyt in pythonToThriftMap:
             return pythonToThriftMap[pyt]
-        elif pyt.__module__.startswith('dynamicserialize.dstypes'):
+        elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'):
             return pythonToThriftMap[types.InstanceType]
         else:
-            raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt))    
+            raise dynamicserialize.SerializationException(
+                "Don't know how to serialize object of type: " + str(pyt))
 
     def serializeMessage(self, obj):
         tt = self._lookupType(obj)
 
         if tt == TType.STRUCT:
-            fqn = obj.__module__.replace('dynamicserialize.dstypes.', '')
+            fqn = obj.__module__[DS_LEN:]
-            if adapters.classAdapterRegistry.has_key(fqn):
+            if fqn in adapters.classAdapterRegistry:
                 # get proper class name when writing class name to serialization stream
                 # in case we have a special inner-class case
                 m = sys.modules[adapters.classAdapterRegistry[fqn].__name__]
@@ -273,7 +277,7 @@ class ThriftSerializationContext(object):
                         val = m[1]()
                         ft = self._lookupType(val)
                         if ft == TType.STRUCT:
-                            fc = val.__module__.replace('dynamicserialize.dstypes.', '')
+                            fc = val.__module__[DS_LEN:]
                             self._serializeField(fieldname, ft, fid, val)
                         else:
                             self._serializeField(fieldname, ft, fid, val)
@@ -293,10 +297,11 @@ class ThriftSerializationContext(object):
         self.protocol.writeFieldEnd()
 
     def _serializeType(self, fieldValue, fieldType):
-        if self.typeSerializationMethod.has_key(fieldType):
+        if fieldType in self.typeSerializationMethod:
             return self.typeSerializationMethod[fieldType](fieldValue)
         else:
-            raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType))
+            raise dynamicserialize.SerializationException(
+                "Unsupported type value " + str(fieldType))
 
     def _serializeArray(self, obj):
         size = len(obj)
@@ -331,7 +336,6 @@ class ThriftSerializationContext(object):
             self.listSerializationMethod[t](obj)
         self.protocol.writeListEnd()
 
-    
     def _serializeMap(self, obj):
         size = len(obj)
         self.protocol.writeMapBegin(TType.VOID, TType.VOID, size)
@@ -418,4 +422,3 @@ class ThriftSerializationContext(object):
 
     def writeObject(self, obj):
         self.serializeMessage(obj)
-    

pythonPackages/dynamicserialize/adapters/__init__.py

@@ -22,6 +22,19 @@
 #
 # __init__.py for Dynamic Serialize adapters.
 #
+# Plugins can contribute to dynamicserialize.adapters by either including their
+# classes directly in pythonPackages/dynamicserialize/adapters/ within their
+# plugin. The plugin's adapter will automatically be added to __all__ at runtime
+# and registered.
+# Plugins should not include a custom __init__.py in 
+# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file. 
+# If custom package initialization is needed, a subpackage should be created
+# with an __init__.py that includes the following:
+#
+# __all__ = ['CustomAdapter1', 'CustomAdapter2']
+# from dynamicserialize.adapters import registerAdapters
+# registerAdapters(__name__, __all__)
+#
 #
 #     SOFTWARE HISTORY
 #
@@ -34,43 +47,54 @@
 #    06/22/2015         #4573      randerso       Added JobProgressAdapter
 #    09/21/2015         #4486      rjpeter        Added FormattedDateAdapter
 #    06/23/2016         #5696      rjpeter        Added CommutativeTimestampAdapter
+#    10/17/2016         #5919      njensen        Added GeomDataRespAdapter
+#    01/09/2017         #5997      nabowle        Allow contribution from plugins.
 #
 
 __all__ = [
     'PointAdapter',
     'StackTraceElementAdapter',
-           'WsIdAdapter',
     'CalendarAdapter',
     'GregorianCalendarAdapter',
-           'ActiveTableModeAdapter',
     'DateAdapter',
-           'FormattedDateAdapter',
-           'LocalizationLevelSerializationAdapter',
-           'LocalizationTypeSerializationAdapter',
     'GeometryTypeAdapter',
     'CoordAdapter',
-           'TimeRangeTypeAdapter',
-           'ParmIDAdapter',
-           'DatabaseIDAdapter',
     'TimestampAdapter',
-           'CommutativeTimestampAdapter',
     'EnumSetAdapter',
     'FloatBufferAdapter',
     'ByteBufferAdapter',
-           'TimeConstraintsAdapter',
+    'JTSEnvelopeAdapter'
-           'LockTableAdapter',
-           'JTSEnvelopeAdapter',
-           'JobProgressAdapter',
 ]
 
 classAdapterRegistry = {}
 
 
 def getAdapterRegistry():
+    import pkgutil
+    
+    discoveredPackages = []
+    # allow other plugins to contribute to adapters by dropping their adapter or
+    # package into the dynamicserialize.adapters package
+    for _, modname, ispkg in pkgutil.iter_modules(__path__):
+        if ispkg:
+            discoveredPackages.append(modname)
+        else:
+            if modname not in __all__:
+                __all__.append(modname)
+    
+    registerAdapters(__name__, __all__)
+    
+    for pkg in discoveredPackages:
+        __import__(__name__ + '.' + pkg)
+    
+            
+def registerAdapters(package, modules):
     import sys
-    for x in __all__:
+    if not package.endswith('.'):
-        exec 'import ' + x
+        package += '.'
-        m = sys.modules['dynamicserialize.adapters.' + x]
+    for x in modules:
+        exec 'import ' + package + x
+        m = sys.modules[package + x]
         d = m.__dict__
         if d.has_key('ClassAdapter'):
             if isinstance(m.ClassAdapter, list):
@@ -80,9 +104,8 @@ def getAdapterRegistry():
                 clzName = m.ClassAdapter
                 classAdapterRegistry[clzName] = m
         else:
-            raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \
+            raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' +
                               'and cannot be registered.')
 
 
 getAdapterRegistry()
-

pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py

@@ -21,22 +21,7 @@
 # File auto-generated by PythonFileGenerator
 
 __all__ = [
-            'activetable',
+            'dataplugin'
-            'alertviz',
-            'auth',
-            'dataaccess',
-            'dataplugin',
-            'dataquery',
-            'datastorage',
-            'localization',
-            'management',
-            'message',
-            'plugin',
-            'pointdata',
-            'pypies',
-            'serialization',
-            'site',
-            'time'
           ]
 
 

pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py

@@ -21,13 +21,7 @@
 # File auto-generated by PythonFileGenerator
 
 __all__ = [
-            'events',
+            'events'
-            'gfe',
-            'grid',
-            'level',
-            'message',
-            'radar',
-            'text'
           ]
 
 

pythonPackages/ufpy/NotificationMessage.py

@@ -42,7 +42,7 @@ from dynamicserialize import DynamicSerializationManager
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    09/30/08                      chammack       Initial Creation.
-#    11/03/10        5849          cjeanbap       Moved to awips package from
+#    11/03/10        5849          cjeanbap       Moved to ufpy package from
 #                                                     com.raytheon.uf.tools.cli
 #    01/07/11        5645          cjeanbap       Added audio file to Status Message.
 #    05/27/11        3050          cjeanbap       Added if-statement to check Priority

pythonPackages/ufpy/QpidSubscriber.py

@@ -30,9 +30,12 @@
 #    11/17/10                      njensen       Initial Creation.
 #    08/15/13        2169          bkowal        Optionally gzip decompress any data that is read.
 #    08/04/16        2416          tgurney       Add queueStarted property
+#    02/16/17        6084          bsteffen      Support ssl connections
 # 
 #
 
+import os
+import os.path
 import qpid
 import zlib
 
@@ -41,11 +44,24 @@ from qpid.exceptions import Closed
 
 class QpidSubscriber:
     
-    def __init__(self, host='127.0.0.1', port=5672, decompress=False):
+    def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None):
         self.host = host
         self.port = port
         self.decompress = decompress;
         socket = qpid.util.connect(host, port)
+        if "QPID_SSL_CERT_DB" in os.environ:
+            certdb = os.environ["QPID_SSL_CERT_DB"]
+        else:
+            certdb = os.path.expanduser("~/.qpid/")
+        if "QPID_SSL_CERT_NAME" in os.environ:
+            certname = os.environ["QPID_SSL_CERT_NAME"]
+        else:
+            certname = "guest" 
+        certfile = os.path.join(certdb, certname + ".crt")
+        if ssl or (ssl is None and os.path.exists(certfile)):
+            keyfile = os.path.join(certdb, certname + ".key")
+            trustfile = os.path.join(certdb, "root.crt")
+            socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
         self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest')
         self.__connection.start()
         self.__session = self.__connection.session(str(qpid.datatypes.uuid4()))

pythonPackages/ufpy/UsageArgumentParser.py

@@ -17,13 +17,23 @@
 # See the AWIPS II Master Rights File ("Master Rights File.pdf") for
 # further licensing information.
 ##
+#
+# SOFTWARE HISTORY
+#
+# Date          Ticket#  Engineer  Description
+# ------------- -------- --------- ---------------------------------------------
+# Feb 13, 2017  6092     randerso  Added StoreTimeAction
+#
+##
 
 import argparse
 import sys
+import time
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID
 
+TIME_FORMAT = "%Y%m%d_%H%M"
 
 class UsageArgumentParser(argparse.ArgumentParser):
     """
@@ -56,3 +66,16 @@ class AppendParmNameAndLevelAction(argparse.Action):
         else:
             setattr(namespace, self.dest, [comp])
 
+class StoreTimeAction(argparse.Action):
+    """
+    argparse.Action subclass to validate GFE formatted time strings
+    and parse them to time.struct_time
+    """
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            timeStruct = time.strptime(values, TIME_FORMAT)
+        except:
+            parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm")
+
+        setattr(namespace, self.dest, timeStruct)
+

pythonPackages/ufpy/__init__.py

@@ -20,7 +20,7 @@
 
 
 #
-# __init__.py for awips package
+# __init__.py for ufpy package
 #  
 #    
 #     SOFTWARE HISTORY

pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py

@@ -31,7 +31,7 @@
 #    06/22/16         #5591        bsteffen       Initial Creation.
 #
 
-from awips.dataaccess import DataAccessLayer
+from ufpy.dataaccess import DataAccessLayer
 
 def getAvailableTimes(request, refTimeOnly=False):
     return __getAvailableTimesForEachParameter(request, refTimeOnly)

pythonPackages/ufpy/dataaccess/DataAccessLayer.py

@@ -20,7 +20,7 @@
 
 
 #
-# Published interface for awips.dataaccess package
+# Published interface for ufpy.dataaccess package
 #
 #
 #     SOFTWARE HISTORY
@@ -40,7 +40,7 @@
 #    Jun 01, 2016    5587          tgurney        Add new signatures for
 #                                                 getRequiredIdentifiers() and
 #                                                 getOptionalIdentifiers()
-#
+#    Oct 18, 2016    5916          bsteffen       Add setLazyLoadGridLatLon
 #
 #
 
@@ -64,7 +64,7 @@ if sys.modules.has_key('jep'):
     import JepRouter
     router = JepRouter
 else:
-    from awips.dataaccess import ThriftClientRouter
+    from ufpy.dataaccess import ThriftClientRouter
     router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
     USING_NATIVE_THRIFT = True
 
@@ -251,3 +251,26 @@ def changeEDEXHost(newHostName):
         router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
     else:
         raise TypeError("Cannot call changeEDEXHost when using JepRouter.")
+
+def setLazyLoadGridLatLon(lazyLoadGridLatLon):
+    """
+    Provide a hint to the Data Access Framework indicating whether to load the
+    lat/lon data for a grid immediately or wait until it is needed. This is
+    provided as a performance tuning hint and should not affect the way the
+    Data Access Framework is used. Depending on the internal implementation of
+    the Data Access Framework this hint might be ignored. Examples of when this
+    should be set to True are when the lat/lon information is not used or when
+    it is used only if certain conditions within the data are met. It could be
+    set to False if it is guaranteed that all lat/lon information is needed and
+    it would be better to get any performance overhead for generating the
+    lat/lon data out of the way during the initial request.
+    
+
+    Args:
+            lazyLoadGridLatLon: Boolean value indicating whether to lazy load.
+    """
+    try:
+        router.setLazyLoadGridLatLon(lazyLoadGridLatLon)
+    except AttributeError:
+        # The router is not required to support this capability.
+        pass

pythonPackages/ufpy/dataaccess/DataNotificationLayer.py

@@ -19,7 +19,7 @@
 # #
 
 #
-# Published interface for retrieving data updates via awips.dataaccess package
+# Published interface for retrieving data updates via ufpy.dataaccess package
 #
 #
 #     SOFTWARE HISTORY
@@ -37,7 +37,7 @@ retrieval of new data as it is coming into the system.
 
 There are two ways to access this feature:
 
-1. The DataQueue module (awips.dataaccess.DataQueue) offers a collection that
+1. The DataQueue module (ufpy.dataaccess.DataQueue) offers a collection that
 automatically fills up with new data as it receives notifications. See that
 module for more information.
 
@@ -49,8 +49,8 @@ each time new data is received.
 Example code follows. This example prints temperature as observed from KOMA
 each time a METAR is received from there.
 
-  from awips.dataaccess import DataAccessLayer as DAL
+  from ufpy.dataaccess import DataAccessLayer as DAL
-  from awips.dataaccess import DataNotificationLayer as DNL
+  from ufpy.dataaccess import DataNotificationLayer as DNL
 
   def process_obs(list_of_data):
       for item in list_of_data:
@@ -69,8 +69,8 @@ each time a METAR is received from there.
 import re
 import sys
 import subprocess
-from awips.dataaccess.PyGeometryNotification import PyGeometryNotification
+from ufpy.dataaccess.PyGeometryNotification import PyGeometryNotification
-from awips.dataaccess.PyGridNotification import PyGridNotification
+from ufpy.dataaccess.PyGridNotification import PyGridNotification
 
 
 THRIFT_HOST = subprocess.check_output(
@@ -89,7 +89,7 @@ if sys.modules.has_key('jep'):
     import JepRouter
     router = JepRouter
 else:
-    from awips.dataaccess import ThriftClientRouter
+    from ufpy.dataaccess import ThriftClientRouter
     router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST)
     USING_NATIVE_THRIFT = True
 

pythonPackages/ufpy/dataaccess/DataQueue.py

@@ -33,7 +33,7 @@
 #    07/29/16        2416          tgurney        Initial creation
 #
 
-from awips.dataaccess import DataNotificationLayer as DNL
+from ufpy.dataaccess import DataNotificationLayer as DNL
 
 import time
 from threading import Thread

pythonPackages/ufpy/dataaccess/PyData.py

@@ -31,7 +31,7 @@
 #    
 #
 
-from awips.dataaccess import IData
+from ufpy.dataaccess import IData
 
 class PyData(IData):
     

pythonPackages/ufpy/dataaccess/PyGeometryData.py

@@ -36,8 +36,8 @@
 #    
 #
 
-from awips.dataaccess import IGeometryData
+from ufpy.dataaccess import IGeometryData
-from awips.dataaccess import PyData
+from ufpy.dataaccess import PyData
 
 class PyGeometryData(IGeometryData, PyData.PyData):
     

pythonPackages/ufpy/dataaccess/PyGeometryNotification.py

@@ -29,7 +29,7 @@
 #    07/22/16        2416          tgurney        Initial creation
 #
 
-from awips.dataaccess.PyNotification import PyNotification
+from ufpy.dataaccess.PyNotification import PyNotification
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 class PyGeometryNotification(PyNotification):

pythonPackages/ufpy/dataaccess/PyGridData.py

@@ -28,7 +28,9 @@
 #    Date            Ticket#       Engineer       Description
 #    ------------    ----------    -----------    --------------------------
 #    06/03/13         #2023        dgilling      Initial Creation.
+#    10/13/16         #5916        bsteffen      Correct grid shape, allow lat/lon
 #    11/10/16         #5900        bsteffen      Correct grid shape
+#                                                to be requested by a delegate
 #    
 #
 
@@ -36,8 +38,8 @@
 import numpy
 import warnings
 
-from awips.dataaccess import IGridData
+from ufpy.dataaccess import IGridData
-from awips.dataaccess import PyData
+from ufpy.dataaccess import PyData
 
 NO_UNIT_CONVERT_WARNING = """
 The ability to unit convert grid data is not currently available in this version of the Data Access Framework.
@@ -46,7 +48,7 @@ The ability to unit convert grid data is not currently available in this version
 
 class PyGridData(IGridData, PyData.PyData):
     
-    def __init__(self, gridDataRecord, nx, ny, latLonGrid):
+    def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None):
         PyData.PyData.__init__(self, gridDataRecord)
         nx = nx
         ny = ny
@@ -54,6 +56,8 @@ class PyGridData(IGridData, PyData.PyData):
         self.__unit = gridDataRecord.getUnit()
         self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx))
         self.__latLonGrid = latLonGrid
+        self.__latLonDelegate = latLonDelegate
+
     
     def getParameter(self):
         return self.__parameter
@@ -70,4 +74,8 @@ class PyGridData(IGridData, PyData.PyData):
         return self.__gridData
     
     def getLatLonCoords(self):
+        if self.__latLonGrid is not None:
+            return self.__latLonGrid
+        elif self.__latLonDelegate is not None:
+            return self.__latLonDelegate()
         return self.__latLonGrid

pythonPackages/ufpy/dataaccess/PyGridNotification.py

@@ -29,7 +29,7 @@
 #    06/03/16        2416          rjpeter       Initial Creation.
 #
 
-from awips.dataaccess.PyNotification import PyNotification
+from ufpy.dataaccess.PyNotification import PyNotification
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 class PyGridNotification(PyNotification):

pythonPackages/ufpy/dataaccess/PyNotification.py

@@ -37,10 +37,10 @@ import time
 import traceback
 
 import dynamicserialize
-from awips.dataaccess import DataAccessLayer
+from ufpy.dataaccess import DataAccessLayer
-from awips.dataaccess import INotificationSubscriber
+from ufpy.dataaccess import INotificationSubscriber
-from awips.QpidSubscriber import QpidSubscriber
+from ufpy.QpidSubscriber import QpidSubscriber
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
 
 

pythonPackages/ufpy/dataaccess/SoundingsSupport.py

@@ -34,8 +34,8 @@
 from collections import defaultdict
 from shapely.geometry import Point
 
-from awips import DateTimeConverter
+from ufpy import DateTimeConverter
-from awips.dataaccess import DataAccessLayer
+from ufpy.dataaccess import DataAccessLayer
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level

pythonPackages/ufpy/dataaccess/ThriftClientRouter.py

@@ -39,7 +39,8 @@
 #                                                 getRequiredIdentifiers() and
 #                                                 getOptionalIdentifiers()
 #    08/01/16        2416          tgurney        Add getNotificationFilter()
-#    11/10/16        5900          bsteffen       Correct grid shape
+#    10/13/16        5916          bsteffen       Correct grid shape, allow lazy grid lat/lon
+#    10/26/16        5919          njensen        Speed up geometry creation in getGeometryData()
 #
 
 
@@ -51,6 +52,7 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest
@@ -59,15 +61,44 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetNotificationFilterRequest
 
-from awips import ThriftClient
+from ufpy import ThriftClient
-from awips.dataaccess import PyGeometryData
+from ufpy.dataaccess import PyGeometryData
-from awips.dataaccess import PyGridData
+from ufpy.dataaccess import PyGridData
+
+
+class LazyGridLatLon(object):
+
+    def __init__(self, client, nx, ny, envelope, crsWkt):
+        self._latLonGrid = None
+        self._client = client
+        self._request = GetGridLatLonRequest()
+        self._request.setNx(nx)
+        self._request.setNy(ny)
+        self._request.setEnvelope(envelope)
+        self._request.setCrsWkt(crsWkt)
+
+    def __call__(self):
+        # Its important that the data is cached internally so that if multiple
+        # GridData are sharing the same delegate then they can also share a
+        # single request for the LatLon information.
+        if self._latLonGrid is None:
+            response = self._client.sendRequest(self._request)
+            nx = response.getNx()
+            ny = response.getNy()
+            latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx))
+            lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx))
+            self._latLonGrid = (lonData, latData)
+        return self._latLonGrid
 
 
 class ThriftClientRouter(object):
 
     def __init__(self, host='localhost'):
         self._client = ThriftClient.ThriftClient(host)
+        self._lazyLoadGridLatLon = False
+
+    def setLazyLoadGridLatLon(self, lazyLoadGridLatLon):
+        self._lazyLoadGridLatLon = lazyLoadGridLatLon
 
     def getAvailableTimes(self, request, refTimeOnly):
         timesRequest = GetAvailableTimesRequest()
@@ -78,6 +109,7 @@ class ThriftClientRouter(object):
 
     def getGridData(self, request, times):
         gridDataRequest = GetGridDataRequest()
+        gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon)
         gridDataRequest.setRequestParameters(request)
         # if we have an iterable times instance, then the user must have asked
         # for grid data with the List of DataTime objects
@@ -95,15 +127,28 @@ class ThriftClientRouter(object):
         for location in locNames:
             nx = response.getSiteNxValues()[location]
             ny = response.getSiteNyValues()[location]
-            latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx))
+            if self._lazyLoadGridLatLon:
-            lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx))
+                envelope = response.getSiteEnvelopes()[location]
+                crsWkt = response.getSiteCrsWkt()[location]
+                delegate = LazyGridLatLon(
+                    self._client, nx, ny, envelope, crsWkt)
+                locSpecificData[location] = (nx, ny, delegate)
+            else:
+                latData = numpy.reshape(numpy.array(
+                    response.getSiteLatGrids()[location]), (ny, nx))
+                lonData = numpy.reshape(numpy.array(
+                    response.getSiteLonGrids()[location]), (ny, nx))
                 locSpecificData[location] = (nx, ny, (lonData, latData))
-
         retVal = []
         for gridDataRecord in response.getGridData():
             locationName = gridDataRecord.getLocationName()
             locData = locSpecificData[locationName]
-            retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2]))
+            if self._lazyLoadGridLatLon:
+                retVal.append(PyGridData.PyGridData(gridDataRecord, locData[
+                              0], locData[1], latLonDelegate=locData[2]))
+            else:
+                retVal.append(PyGridData.PyGridData(
+                    gridDataRecord, locData[0], locData[1], locData[2]))
         return retVal
 
     def getGeometryData(self, request, times):
@@ -121,10 +166,9 @@ class ThriftClientRouter(object):
         response = self._client.sendRequest(geoDataRequest)
         geometries = []
         for wkb in response.getGeometryWKBs():
-            # convert the wkb to a bytearray with only positive values
+            # the wkb is a numpy.ndarray of dtype int8
-            byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist()))
+            # convert the bytearray to a byte string and load it
-            # convert the bytearray to a byte string and load it.
+            geometries.append(shapely.wkb.loads(wkb.tostring()))
-            geometries.append(shapely.wkb.loads(str(byteArrWKB)))
 
         retVal = []
         for geoDataRecord in response.getGeoData():

pythonPackages/ufpy/dataaccess/__init__.py

@@ -20,7 +20,7 @@
 
 
 #
-# __init__.py for awips.dataaccess package
+# __init__.py for ufpy.dataaccess package
 #
 #
 #     SOFTWARE HISTORY

pythonPackages/ufpy/gfe/IFPClient.py

@@ -18,7 +18,7 @@
 # further licensing information.
 ##
 
-from awips import ThriftClient
+from ufpy import ThriftClient
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID

pythonPackages/ufpy/gfe/__init__.py

@@ -20,7 +20,7 @@
 
 
 #
-# __init__.py for awips.gfe package
+# __init__.py for ufpy.gfe package
 #  
 #    
 #     SOFTWARE HISTORY

pythonPackages/ufpy/qpidingest.py

@@ -59,9 +59,13 @@
 #    ....
 #    06/13/2013      DR 16242      D. Friedman    Add Qpid authentication info
 #    03/06/2014      DR 17907      D. Friedman    Workaround for issue QPID-5569
+#    02/16/2017      DR 6084       bsteffen       Support ssl connections
 #
 #===============================================================================   
 
+import os
+import os.path
+
 import qpid
 from qpid.util import connect
 from qpid.connection import Connection
@@ -71,17 +75,31 @@ QPID_USERNAME = 'guest'
 QPID_PASSWORD = 'guest'
 
 class IngestViaQPID:
-    def __init__(self, host='localhost', port=5672):
+    def __init__(self, host='localhost', port=5672, ssl=None):
         '''
         Connect to QPID and make bindings to route message to external.dropbox queue
         @param host: string hostname of computer running EDEX and QPID (default localhost)
         @param port: integer port used to connect to QPID (default 5672)
+        @param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found.
         '''
         
         try:
             #
-            self.socket = connect(host, port)
+            socket = connect(host, port)
-            self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD)
+            if "QPID_SSL_CERT_DB" in os.environ:
+                certdb = os.environ["QPID_SSL_CERT_DB"]
+            else:
+                certdb = os.path.expanduser("~/.qpid/")
+            if "QPID_SSL_CERT_NAME" in os.environ:
+                certname = os.environ["QPID_SSL_CERT_NAME"]
+            else:
+                certname = QPID_USERNAME
+            certfile = os.path.join(certdb, certname + ".crt")
+            if ssl or (ssl is None and os.path.exists(certfile)):
+                keyfile = os.path.join(certdb, certname + ".key")
+                trustfile = os.path.join(certdb, "root.crt")
+                socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile)
+            self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD)
             self.connection.start()
             self.session = self.connection.session(str(uuid4()))
             self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox')

pythonPackages/ufpy/test/Test

@@ -39,7 +39,7 @@
 
 import os
 import logging
-from awips import AlertVizHandler
+from ufpy import AlertVizHandler
 import Record
 
 avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET)

pythonPackages/ufpy/test/__init__.py

@@ -20,7 +20,7 @@
 
 
 #
-# __init__.py for awips package
+# __init__.py for ufpy package
 #  
 #    
 #     SOFTWARE HISTORY

pythonPackages/ufpy/test/dafTests/__init__.py

@@ -20,7 +20,7 @@
 
 
 #
-# __init__.py for awips.test.dafTests package
+# __init__.py for ufpy.test.dafTests package
 #
 #
 #     SOFTWARE HISTORY

pythonPackages/ufpy/test/dafTests/baseBufrMosTestCase.py

@@ -18,9 +18,12 @@
 # further licensing information.
 ##
 
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
+from shapely.geometry import box
 
 import baseDafTestCase
+import params
+import unittest
 
 #
 # Base TestCase for BufrMos* tests.
@@ -31,7 +34,8 @@ import baseDafTestCase
 #    ------------    ----------    -----------    --------------------------
 #    01/19/16        4795          mapeters       Initial Creation.
 #    04/11/16        5548          tgurney        Cleanup
-#
+#    12/07/16        5981          tgurney        Parameterize
+#    12/15/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -39,6 +43,8 @@ import baseDafTestCase
 class BufrMosTestCase(baseDafTestCase.DafTestCase):
     """Base class for testing DAF support of bufrmos data"""
     
+    data_params = "temperature", "dewpoint"
+    
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
         self.runParametersTest(req)
@@ -49,11 +55,19 @@ class BufrMosTestCase(baseDafTestCase.DafTestCase):
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
         self.runTimesTest(req)
 
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
-        req.setParameters("temperature", "dewpoint")
+        req.setParameters(*self.data_params)
         self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters(*self.data_params)
+        req.setEnvelope(params.ENVELOPE)
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))

pythonPackages/ufpy/test/dafTests/baseDafTestCase.py

@@ -20,8 +20,8 @@
 
 from __future__ import print_function
 
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 import os
 import unittest
@@ -50,6 +50,8 @@ import unittest
 #    10/05/16        5926          dgilling       Better checks in runGeometryDataTest.
 #    11/08/16        5985          tgurney        Do not check data times on
 #                                                 time-agnostic data
+#    03/13/17        5981          tgurney        Do not check valid period on
+#                                                 data time
 #
 #
 
@@ -166,10 +168,13 @@ class DafTestCase(unittest.TestCase):
         self.assertIsNotNone(geomData)
         if times:
             self.assertNotEqual(len(geomData), 0)
+        if not geomData:
+            raise unittest.SkipTest("No data available")
         print("Number of geometry records: " + str(len(geomData)))
         print("Sample geometry data:")
         for record in geomData[:self.sampleDataLimit]:
-            if checkDataTimes and times:
+            if (checkDataTimes and times and
+                    "PERIOD_USED" not in record.getDataTime().getUtilityFlags()):
                 self.assertIn(record.getDataTime(), times[:self.numTimesToLimit])
             print("geometry=" + str(record.getGeometry()), end="")
             for p in req.getParameters():
@@ -184,6 +189,8 @@ class DafTestCase(unittest.TestCase):
         """
         geomData = DAL.getGeometryData(req, timeRange)
         self.assertIsNotNone(geomData)
+        if not geomData:
+            raise unittest.SkipTest("No data available")
         print("Number of geometry records: " + str(len(geomData)))
         print("Sample geometry data:")
         for record in geomData[:self.sampleDataLimit]:
@@ -207,6 +214,8 @@ class DafTestCase(unittest.TestCase):
         times = DafTestCase.getTimesIfSupported(req)
         gridData = DAL.getGridData(req, times[:self.numTimesToLimit])
         self.assertIsNotNone(gridData)
+        if not gridData:
+            raise unittest.SkipTest("No data available")
         print("Number of grid records: " + str(len(gridData)))
         if len(gridData) > 0:
             print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")

pythonPackages/ufpy/test/dafTests/baseRadarTestCase.py

@@ -0,0 +1,194 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from __future__ import print_function
+from shapely.geometry import box
+from ufpy.dataaccess import DataAccessLayer as DAL
+from ufpy.ThriftClient import ThriftRequestException
+
+import baseDafTestCase
+import params
+import unittest
+
+#
+# Tests common to all radar factories
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    01/19/16        4795          mapeters       Initial Creation.
+#    04/11/16        5548          tgurney        Cleanup
+#    04/18/16        5548          tgurney        More cleanup
+#    04/26/16        5587          tgurney        Move identifier values tests
+#                                                 out of base class
+#    06/01/16        5587          tgurney        Update testGetIdentifierValues
+#    06/08/16        5574          mapeters       Add advanced query tests
+#    06/13/16        5574          tgurney        Fix checks for None
+#    06/14/16        5548          tgurney        Undo previous change (broke
+#                                                 test)
+#    06/30/16        5725          tgurney        Add test for NOT IN
+#    08/25/16        2671          tgurney        Rename to baseRadarTestCase
+#                                                 and move factory-specific
+#                                                 tests
+#    12/07/16        5981          tgurney        Parameterize
+#
+#
+
+
+class BaseRadarTestCase(baseDafTestCase.DafTestCase):
+    """Tests common to all radar factories"""
+
+    # datatype is specified by subclass
+    datatype = None
+
+    radarLoc = params.RADAR.lower()
+
+    def testGetAvailableParameters(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runParametersTest(req)
+
+    def testGetAvailableLocations(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runLocationsTest(req)
+
+    def testGetAvailableLevels(self):
+        req = DAL.newDataRequest(self.datatype)
+        self.runLevelsTest(req)
+        
+    def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('level.one.field', 'invalidLevelField')
+        with self.assertRaises(ThriftRequestException) as cm:
+            self.runLevelsTest(req)
+        self.assertIn('IncompatibleRequestException', str(cm.exception))
+
+    def testGetAvailableTimes(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        self.runTimesTest(req)
+
+    def testGetIdentifierValues(self):
+        req = DAL.newDataRequest(self.datatype)
+        optionalIds = set(DAL.getOptionalIdentifiers(req))
+        requiredIds = set(DAL.getRequiredIdentifiers(req))
+        self.runGetIdValuesTest(optionalIds | requiredIds)
+
+    def testGetInvalidIdentifierValuesThrowsException(self):
+        self.runInvalidIdValuesTest()
+
+    def testGetNonexistentIdentifierValuesThrowsException(self):
+        self.runNonexistentIdValuesTest()
+
+    def runConstraintTest(self, key, operator, value):
+        raise NotImplementedError
+
+    def testGetDataWithEqualsString(self):
+        gridData = self.runConstraintTest('icao', '=', self.radarLoc)
+        for record in gridData:
+            self.assertEqual(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithEqualsUnicode(self):
+        gridData = self.runConstraintTest('icao', '=', unicode(self.radarLoc))
+        for record in gridData:
+            self.assertEqual(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithEqualsInt(self):
+        gridData = self.runConstraintTest('icao', '=', 1000)
+        for record in gridData:
+            self.assertEqual(record.getAttribute('icao'), 1000)
+
+    def testGetDataWithEqualsLong(self):
+        gridData = self.runConstraintTest('icao', '=', 1000L)
+        for record in gridData:
+            self.assertEqual(record.getAttribute('icao'), 1000)
+
+    def testGetDataWithEqualsFloat(self):
+        gridData = self.runConstraintTest('icao', '=', 1.0)
+        for record in gridData:
+            self.assertEqual(round(record.getAttribute('icao'), 1), 1.0)
+
+    def testGetDataWithEqualsNone(self):
+        gridData = self.runConstraintTest('icao', '=', None)
+        for record in gridData:
+            self.assertIsNone(record.getAttribute('icao'))
+
+    def testGetDataWithNotEquals(self):
+        gridData = self.runConstraintTest('icao', '!=', self.radarLoc)
+        for record in gridData:
+            self.assertNotEqual(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithNotEqualsNone(self):
+        gridData = self.runConstraintTest('icao', '!=', None)
+        for record in gridData:
+            self.assertIsNotNone(record.getAttribute('icao'))
+
+    def testGetDataWithGreaterThan(self):
+        gridData = self.runConstraintTest('icao', '>', self.radarLoc)
+        for record in gridData:
+            self.assertGreater(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithLessThan(self):
+        gridData = self.runConstraintTest('icao', '<', self.radarLoc)
+        for record in gridData:
+            self.assertLess(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithGreaterThanEquals(self):
+        gridData = self.runConstraintTest('icao', '>=', self.radarLoc)
+        for record in gridData:
+            self.assertGreaterEqual(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithLessThanEquals(self):
+        gridData = self.runConstraintTest('icao', '<=', self.radarLoc)
+        for record in gridData:
+            self.assertLessEqual(record.getAttribute('icao'), self.radarLoc)
+
+    def testGetDataWithInTuple(self):
+        gridData = self.runConstraintTest('icao', 'in', (self.radarLoc, 'tpbi'))
+        for record in gridData:
+            self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi'))
+
+    def testGetDataWithInList(self):
+        gridData = self.runConstraintTest('icao', 'in', [self.radarLoc, 'tpbi'])
+        for record in gridData:
+            self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi'))
+
+    def testGetDataWithInGenerator(self):
+        generator = (item for item in (self.radarLoc, 'tpbi'))
+        gridData = self.runConstraintTest('icao', 'in', generator)
+        for record in gridData:
+            self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi'))
+
+    def testGetDataWithNotInList(self):
+        gridData = self.runConstraintTest('icao', 'not in', ['zzzz', self.radarLoc])
+        for record in gridData:
+            self.assertNotIn(record.getAttribute('icao'), ('zzzz', self.radarLoc))
+
+    def testGetDataWithInvalidConstraintTypeThrowsException(self):
+        with self.assertRaises(ValueError):
+            self.runConstraintTest('icao', 'junk', self.radarLoc)
+
+    def testGetDataWithInvalidConstraintValueThrowsException(self):
+        with self.assertRaises(TypeError):
+            self.runConstraintTest('icao', '=', {})
+
+    def testGetDataWithEmptyInConstraintThrowsException(self):
+        with self.assertRaises(ValueError):
+            self.runConstraintTest('icao', 'in', [])

pythonPackages/ufpy/test/dafTests/params.py

@@ -0,0 +1,43 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+
+#
+# Site-specific parameters for DAF tests
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    12/07/16        5981          tgurney        Initial creation
+#    12/15/16        5981          tgurney        Add ENVELOPE
+#
+#
+
+from shapely.geometry import box
+
+AIRPORT = 'OMA'
+OBS_STATION = 'KOMA'
+SITE_ID = 'OAX'
+STATION_ID = '72558'
+RADAR = 'KOAX'
+SAMPLE_AREA = (-97.0, 41.0, -96.0, 42.0)
+
+ENVELOPE = box(*SAMPLE_AREA)

pythonPackages/ufpy/test/dafTests/testAcars.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
 import unittest

pythonPackages/ufpy/test/dafTests/testAirep.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase

pythonPackages/ufpy/test/dafTests/testBinLightning.py

@@ -19,8 +19,8 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 

pythonPackages/ufpy/test/dafTests/testBufrMosHpc.py

@@ -19,9 +19,10 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseBufrMosTestCase
+import params
 import unittest
 
 #
@@ -34,6 +35,8 @@ import unittest
 #    01/19/16        4795          mapeters       Initial Creation.
 #    04/11/16        5548          tgurney        Cleanup
 #    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Inherit all tests
 #
 #
 
@@ -42,11 +45,6 @@ class BufrMosHpcTestCase(baseBufrMosTestCase.BufrMosTestCase):
     """Test DAF support for bufrmosHPC data"""
 
     datatype = "bufrmosHPC"
+    data_params = "forecastHr", "maxTemp24Hour"
 
-    # Most tests inherited from superclass
+    # All tests inherited from superclass
-
-    def testGetGeometryData(self):
-        req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
-        req.setParameters("forecastHr", "maxTemp24Hour")
-        self.runGeometryDataTest(req)

pythonPackages/ufpy/test/dafTests/testBufrMosMrf.py

@@ -19,9 +19,10 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseBufrMosTestCase
+import params
 import unittest
 
 #
@@ -34,6 +35,8 @@ import unittest
 #    01/19/16        4795          mapeters       Initial Creation.
 #    04/11/16        5548          tgurney        Cleanup
 #    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Inherit all tests
 #
 #
 
@@ -42,11 +45,6 @@ class BufrMosMrfTestCase(baseBufrMosTestCase.BufrMosTestCase):
     """Test DAF support for bufrmosMRF data"""
 
     datatype = "bufrmosMRF"
+    data_params = "forecastHr", "maxTempDay"
 
-    # Most tests inherited from superclass
+    # All tests inherited from superclass
-
-    def testGetGeometryData(self):
-        req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
-        req.setParameters("forecastHr", "maxTempDay")
-        self.runGeometryDataTest(req)

pythonPackages/ufpy/test/dafTests/testBufrUa.py

@@ -19,10 +19,11 @@
 # #
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -38,6 +39,8 @@ import unittest
 #    06/09/16        5587          bsteffen       Add getIdentifierValues tests
 #    06/13/16        5574          tgurney        Add advanced query tests
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    12/07/16        5981          tgurney        Parameterize
+#    12/15/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -47,8 +50,7 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase):
 
     datatype = "bufrua"
 
-    location = "72558"
+    location = params.STATION_ID
-    """stationid corresponding to KOAX"""
 
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
@@ -91,6 +93,14 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase):
 
         print("getGeometryData() complete\n\n")
 
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters("staName", "rptType")
+        req.setEnvelope(params.ENVELOPE)
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
+
     def testGetIdentifierValues(self):
         req = DAL.newDataRequest(self.datatype)
         optionalIds = set(DAL.getOptionalIdentifiers(req))
@@ -133,7 +143,6 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase):
 
     # No float test because no float identifiers are available
 
-
     def testGetDataWithEqualsNone(self):
         geometryData = self._runConstraintTest('reportType', '=', None)
         for record in geometryData:

pythonPackages/ufpy/test/dafTests/testClimate.py

@@ -22,10 +22,11 @@ from __future__ import print_function
 import datetime
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -44,6 +45,8 @@ import unittest
 #    06/21/16        5548          tgurney        Skip tests that cause errors
 #    06/30/16        5725          tgurney        Add test for NOT IN
 #    10/06/16        5926          dgilling       Add additional time and location tests.
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -52,6 +55,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for climate data"""
 
     datatype = 'climate'
+    obsStation = params.OBS_STATION
 
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
@@ -104,7 +108,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         """
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('table', 'public.cli_asos_monthly')
-        req.setLocationNames('KOMA', 'KABR', 'KDMO')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
         req.setParameters('maxtemp_mon', 'min_sea_press')
         self.runTimesTest(req)
         
@@ -115,7 +119,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         """
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('table', 'public.cli_asos_daily')
-        req.setLocationNames('KOMA', 'KABR', 'KDMO')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
         req.setParameters('maxtemp_cal', 'min_press')
         self.runTimesTest(req)
 
@@ -126,7 +130,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         """
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('table', 'public.cli_mon_season_yr')
-        req.setLocationNames('KOMA', 'KABR', 'KDMO')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
         req.setParameters('max_temp', 'precip_total')
         self.runTimesTest(req)
 
@@ -137,7 +141,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         """
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('table', 'public.daily_climate')
-        req.setLocationNames('KOMA', 'KABR', 'KDMO')
+        req.setLocationNames(self.obsStation, 'KABR', 'KDMO')
         req.setParameters('max_temp', 'precip', 'avg_wind_speed')
         self.runTimesTest(req)
 
@@ -155,6 +159,15 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         req.setParameters('maxtemp_mon', 'min_sea_press')
         self.runGeometryDataTest(req)
 
+    def testGetGeometryDataWithEnvelopeThrowsException(self):
+        # Envelope is not used
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('table', 'public.cli_asos_monthly')
+        req.setParameters('maxtemp_mon', 'min_sea_press')
+        req.setEnvelope(params.ENVELOPE)
+        with self.assertRaises(Exception):
+            data = self.runGeometryDataTest(req)
+
     def testGetGeometryDataForYearAndDayOfYearTable(self):
         """
         Test retrieval of data for a climo table that uses year and 
@@ -243,14 +256,14 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         return self.runGeometryDataTest(req)
 
     def testGetDataWithEqualsString(self):
-        geometryData = self._runConstraintTest('station_code', '=', 'KOMA')
+        geometryData = self._runConstraintTest('station_code', '=', self.obsStation)
         for record in geometryData:
-            self.assertEqual(record.getString('station_code'), 'KOMA')
+            self.assertEqual(record.getString('station_code'), self.obsStation)
 
     def testGetDataWithEqualsUnicode(self):
-        geometryData = self._runConstraintTest('station_code', '=', u'KOMA')
+        geometryData = self._runConstraintTest('station_code', '=', unicode(self.obsStation))
         for record in geometryData:
-            self.assertEqual(record.getString('station_code'), 'KOMA')
+            self.assertEqual(record.getString('station_code'), self.obsStation)
 
     def testGetDataWithEqualsInt(self):
         geometryData = self._runConstraintTest('avg_daily_max', '=', 70)
@@ -272,9 +285,9 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         self.assertEqual(len(geometryData), 0)
 
     def testGetDataWithNotEquals(self):
-        geometryData = self._runConstraintTest('station_code', '!=', 'KOMA')
+        geometryData = self._runConstraintTest('station_code', '!=', self.obsStation)
         for record in geometryData:
-            self.assertNotEqual(record.getString('station_code'), 'KOMA')
+            self.assertNotEqual(record.getString('station_code'), self.obsStation)
 
     def testGetDataWithNotEqualsNone(self):
         geometryData = self._runConstraintTest('station_code', '!=', None)
@@ -302,19 +315,19 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
             self.assertLessEqual(record.getNumber('avg_daily_max'), 70)
 
     def testGetDataWithInTuple(self):
-        collection = ('KOMA', 'KABR')
+        collection = (self.obsStation, 'KABR')
         geometryData = self._runConstraintTest('station_code', 'in', collection)
         for record in geometryData:
             self.assertIn(record.getString('station_code'), collection)
 
     def testGetDataWithInList(self):
-        collection = ['KOMA', 'KABR']
+        collection = [self.obsStation, 'KABR']
         geometryData = self._runConstraintTest('station_code', 'in', collection)
         for record in geometryData:
             self.assertIn(record.getString('station_code'), collection)
 
     def testGetDataWithInGenerator(self):
-        collection = ('KOMA', 'KABR')
+        collection = (self.obsStation, 'KABR')
         generator = (item for item in collection)
         geometryData = self._runConstraintTest('station_code', 'in', generator)
         for record in geometryData:
@@ -328,7 +341,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
 
     def testGetDataWithInvalidConstraintTypeThrowsException(self):
         with self.assertRaises(ValueError):
-            self._runConstraintTest('station_code', 'junk', 'KOMA')
+            self._runConstraintTest('station_code', 'junk', self.obsStation)
 
     def testGetDataWithInvalidConstraintValueThrowsException(self):
         with self.assertRaises(TypeError):
@@ -417,4 +430,3 @@ class ClimateTestCase(baseDafTestCase.DafTestCase):
         endTime = datetime.datetime(2009, 3, 31)
         tr = TimeRange(startTime, endTime)
         self.runGeometryDataTestWithTimeRange(req, tr)
-

pythonPackages/ufpy/test/dafTests/testCombinedTimeQuery.py

@@ -18,9 +18,9 @@
 # further licensing information.
 ##
 
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
-from awips.dataaccess import CombinedTimeQuery as CTQ
+from ufpy.dataaccess import CombinedTimeQuery as CTQ
 
 import unittest
 import os
@@ -49,7 +49,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase):
 
     def testSuccessfulQuery(self):
         req = DAL.newDataRequest('grid')
-        req.setLocationNames('RAP13')
+        req.setLocationNames('RUC130')
         req.setParameters('T','GH')
         req.setLevels('300MB', '500MB','700MB')
         times = CTQ.getAvailableTimes(req);
@@ -60,7 +60,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase):
         Test that when a parameter is only available on one of the levels that no times are returned.
         """
         req = DAL.newDataRequest('grid')
-        req.setLocationNames('RAP13')
+        req.setLocationNames('RUC130')
         req.setParameters('T','GH', 'LgSP1hr')
         req.setLevels('300MB', '500MB','700MB','0.0SFC')
         times = CTQ.getAvailableTimes(req);

pythonPackages/ufpy/test/dafTests/testCommonObsSpatial.py

@@ -20,10 +20,11 @@
 
 from __future__ import print_function
 from shapely.geometry import box
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -42,6 +43,8 @@ import unittest
 #    06/13/16        5574          tgurney        Add advanced query tests
 #    06/21/16        5548          tgurney        Skip tests that cause errors
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    12/07/16        5981          tgurney        Parameterize
+#    01/06/17        5981          tgurney        Do not check data times
 #
 
 
@@ -50,9 +53,6 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
 
     datatype = "common_obs_spatial"
 
-    envelope = box(-97.0, 41.0, -96.0, 42.0)
-    """Default request area (box around KOAX)"""
-
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
         self.runParametersTest(req)
@@ -65,19 +65,11 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
     def testGetIdentifierValues(self):
         self.runGetIdValuesTest(['country'])
 
-    @unittest.skip('avoid EDEX error')
-    def testGetInvalidIdentifierValuesThrowsException(self):
-        self.runInvalidIdValuesTest()
-
-    @unittest.skip('avoid EDEX error')
-    def testGetNonexistentIdentifierValuesThrowsException(self):
-        self.runNonexistentIdValuesTest()
-
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setEnvelope(self.envelope)
+        req.setEnvelope(params.ENVELOPE)
         req.setParameters("name", "stationid")
-        self.runGeometryDataTest(req)
+        self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testRequestingTimesThrowsTimeAgnosticDataException(self):
         req = DAL.newDataRequest(self.datatype)
@@ -88,7 +80,7 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase):
         constraint = RequestConstraint.new(operator, value)
         req.addIdentifier(key, constraint)
         req.setParameters('catalogtype', 'elevation', 'state')
-        return self.runGeometryDataTest(req)
+        return self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetDataWithEqualsString(self):
         geometryData = self._runConstraintTest('state', '=', 'NE')

pythonPackages/ufpy/test/dafTests/testFfmp.py

@@ -20,9 +20,10 @@
 
 from __future__ import print_function
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -46,6 +47,8 @@ import unittest
 #                                                 PRTM parameter since it isn't
 #                                                 configured for ec-oma
 #    11/08/16        5985          tgurney        Do not check data times
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Do not check data times
 #
 #
 
@@ -54,10 +57,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
     """Test DAF support for ffmp data"""
 
     datatype = 'ffmp'
+    location = params.RADAR.lower()
 
     @staticmethod
     def addIdentifiers(req):
-        req.addIdentifier('wfo', 'OAX')
+        req.addIdentifier('wfo', params.SITE_ID)
         req.addIdentifier('siteKey', 'hpe')
         req.addIdentifier('dataKey', 'hpe')
         req.addIdentifier('huc', 'ALL')
@@ -99,8 +103,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
             req = DAL.newDataRequest(self.datatype)
             if id == 'accumHrs':
                 req.setParameters('ARI6H2YR')
-                req.addIdentifier('wfo', 'OAX')
+                req.addIdentifier('wfo', params.SITE_ID)
-                req.addIdentifier('siteKey', 'koax')
+                req.addIdentifier('siteKey', self.location)
                 req.addIdentifier('huc', 'ALL')
             idValues = DAL.getIdentifierValues(req, id)
             self.assertTrue(hasattr(idValues, '__iter__'))
@@ -116,20 +120,20 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
         req = DAL.newDataRequest(self.datatype)
         constraint = RequestConstraint.new(operator, value)
         req.addIdentifier(key, constraint)
-        req.addIdentifier('wfo', 'OAX')
+        req.addIdentifier('wfo', params.SITE_ID)
         req.addIdentifier('huc', 'ALL')
         req.setParameters('QPFSCAN')
         return self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetDataWithEqualsString(self):
-        geometryData = self._runConstraintTest('siteKey', '=', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '=', self.location)
         for record in geometryData:
-            self.assertEqual(record.getAttribute('siteKey'), 'koax')
+            self.assertEqual(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithEqualsUnicode(self):
-        geometryData = self._runConstraintTest('siteKey', '=', u'koax')
+        geometryData = self._runConstraintTest('siteKey', '=', unicode(self.location))
         for record in geometryData:
-            self.assertEqual(record.getAttribute('siteKey'), 'koax')
+            self.assertEqual(record.getAttribute('siteKey'), self.location)
 
     # No numeric tests since no numeric identifiers are available that support
     # RequestConstraints.
@@ -140,9 +144,9 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
             self.assertIsNone(record.getAttribute('siteKey'))
 
     def testGetDataWithNotEquals(self):
-        geometryData = self._runConstraintTest('siteKey', '!=', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '!=', self.location)
         for record in geometryData:
-            self.assertNotEqual(record.getAttribute('siteKey'), 'koax')
+            self.assertNotEqual(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithNotEqualsNone(self):
         geometryData = self._runConstraintTest('siteKey', '!=', None)
@@ -150,40 +154,40 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
             self.assertIsNotNone(record.getAttribute('siteKey'))
 
     def testGetDataWithGreaterThan(self):
-        geometryData = self._runConstraintTest('siteKey', '>', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '>', self.location)
         for record in geometryData:
-            self.assertGreater(record.getAttribute('siteKey'), 'koax')
+            self.assertGreater(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithLessThan(self):
-        geometryData = self._runConstraintTest('siteKey', '<', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '<', self.location)
         for record in geometryData:
-            self.assertLess(record.getAttribute('siteKey'), 'koax')
+            self.assertLess(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithGreaterThanEquals(self):
-        geometryData = self._runConstraintTest('siteKey', '>=', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '>=', self.location)
         for record in geometryData:
-            self.assertGreaterEqual(record.getAttribute('siteKey'), 'koax')
+            self.assertGreaterEqual(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithLessThanEquals(self):
-        geometryData = self._runConstraintTest('siteKey', '<=', 'koax')
+        geometryData = self._runConstraintTest('siteKey', '<=', self.location)
         for record in geometryData:
-            self.assertLessEqual(record.getAttribute('siteKey'), 'koax')
+            self.assertLessEqual(record.getAttribute('siteKey'), self.location)
 
     def testGetDataWithInList(self):
-        collection = ['koax', 'kuex']
+        collection = [self.location, 'kuex']
         geometryData = self._runConstraintTest('siteKey', 'in', collection)
         for record in geometryData:
             self.assertIn(record.getAttribute('siteKey'), collection)
 
     def testGetDataWithNotInList(self):
-        collection = ['koax', 'kuex']
+        collection = [self.location, 'kuex']
         geometryData = self._runConstraintTest('siteKey', 'not in', collection)
         for record in geometryData:
             self.assertNotIn(record.getAttribute('siteKey'), collection)
 
     def testGetDataWithInvalidConstraintTypeThrowsException(self):
         with self.assertRaises(ValueError):
-            self._runConstraintTest('siteKey', 'junk', 'koax')
+            self._runConstraintTest('siteKey', 'junk', self.location)
 
     def testGetDataWithInvalidConstraintValueThrowsException(self):
         with self.assertRaises(TypeError):
@@ -194,11 +198,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
             self._runConstraintTest('siteKey', 'in', [])
 
     def testGetDataWithSiteKeyAndDataKeyConstraints(self):
-        siteKeys = ['koax', 'hpe']
+        siteKeys = [self.location, 'hpe']
         dataKeys = ['kuex', 'kdmx']
 
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('wfo', 'OAX')
+        req.addIdentifier('wfo', params.SITE_ID)
         req.addIdentifier('huc', 'ALL')
 
         siteKeysConstraint = RequestConstraint.new('in', siteKeys)
@@ -217,8 +221,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase):
     def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self):
         # Test that accumHrs identifier is not required for guidance data
         req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('wfo', 'OAX')
+        req.addIdentifier('wfo', params.SITE_ID)
-        req.addIdentifier('siteKey', 'koax')
+        req.addIdentifier('siteKey', self.location)
         req.addIdentifier('huc', 'ALL')
         req.setParameters('FFG0124hr')
         self.runGeometryDataTest(req, checkDataTimes=False)

pythonPackages/ufpy/test/dafTests/testGfe.py

@@ -20,9 +20,11 @@
 
 from __future__ import print_function
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
+from shapely.geometry import box, Point
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -41,6 +43,9 @@ import unittest
 #    06/17/16        5574          mapeters       Add advanced query tests
 #    06/30/16        5725          tgurney        Add test for NOT IN
 #    11/07/16        5991          bsteffen       Improve vector tests
+#    12/07/16        5981          tgurney        Parameterize
+#    12/15/16        6040          tgurney        Add testGetGridDataWithDbType
+#    12/20/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -62,20 +67,49 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('modelName', 'Fcst')
-        req.addIdentifier('siteId', 'OAX')
+        req.addIdentifier('siteId', params.SITE_ID)
         self.runTimesTest(req)
 
     def testGetGridData(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('modelName', 'Fcst')
-        req.addIdentifier('siteId', 'OAX')
+        req.addIdentifier('siteId', params.SITE_ID)
         req.setParameters('T')
         self.runGridDataTest(req)
 
+    def testGetGridDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier('modelName', 'Fcst')
+        req.addIdentifier('siteId', params.SITE_ID)
+        req.setParameters('T')
+        req.setEnvelope(params.ENVELOPE)
+        gridData = self.runGridDataTest(req)
+        if not gridData:
+            raise unittest.SkipTest('no data available')
+        lons, lats = gridData[0].getLatLonCoords()
+        lons = lons.reshape(-1)
+        lats = lats.reshape(-1)
+
+        # Ensure all points are within one degree of the original box
+        # to allow slight margin of error for reprojection distortion.
+        testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
+                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
+
+        for i in range(len(lons)):
+            self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))
+    
+    def testGetGridDataWithDbType(self):
+        req = DAL.newDataRequest('gfe')
+        req.addIdentifier('parmId.dbId.modelName', 'Fcst')
+        req.addIdentifier('parmId.dbId.dbType', 'Prac')
+        req.setParameters('T', 'Td')
+        times = DAL.getAvailableTimes(req)
+        self.runGridDataTest(req)
+
     def testGetVectorGridData(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('modelName', 'Fcst')
-        req.addIdentifier('siteId', 'OAX')
+        req.addIdentifier('siteId', params.SITE_ID)
         req.setParameters('Wind')
         times = DAL.getAvailableTimes(req)
         if not(times):
@@ -114,80 +148,80 @@ class GfeTestCase(baseDafTestCase.DafTestCase):
         req = DAL.newDataRequest(self.datatype)
         constraint = RequestConstraint.new(operator, value)
         req.addIdentifier(key, constraint)
-        req.setLocationNames('OAX')
+        req.setLocationNames(params.SITE_ID)
         req.setParameters('T')
         return self.runGridDataTest(req)
 
     def testGetDataWithEqualsString(self):
-        geometryData = self._runConstraintTest('modelName', '=', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '=', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertEqual(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithEqualsUnicode(self):
-        geometryData = self._runConstraintTest('modelName', '=', u'Fcst')
+        gridData = self._runConstraintTest('modelName', '=', u'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertEqual(record.getAttribute('modelName'), 'Fcst')
 
     # No numeric tests since no numeric identifiers are available.
 
     def testGetDataWithEqualsNone(self):
-        geometryData = self._runConstraintTest('modelName', '=', None)
+        gridData = self._runConstraintTest('modelName', '=', None)
-        for record in geometryData:
+        for record in gridData:
             self.assertIsNone(record.getAttribute('modelName'))
 
     def testGetDataWithNotEquals(self):
-        geometryData = self._runConstraintTest('modelName', '!=', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '!=', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertNotEqual(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithNotEqualsNone(self):
-        geometryData = self._runConstraintTest('modelName', '!=', None)
+        gridData = self._runConstraintTest('modelName', '!=', None)
-        for record in geometryData:
+        for record in gridData:
             self.assertIsNotNone(record.getAttribute('modelName'))
 
     def testGetDataWithGreaterThan(self):
-        geometryData = self._runConstraintTest('modelName', '>', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '>', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertGreater(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithLessThan(self):
-        geometryData = self._runConstraintTest('modelName', '<', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '<', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertLess(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithGreaterThanEquals(self):
-        geometryData = self._runConstraintTest('modelName', '>=', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '>=', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertGreaterEqual(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithLessThanEquals(self):
-        geometryData = self._runConstraintTest('modelName', '<=', 'Fcst')
+        gridData = self._runConstraintTest('modelName', '<=', 'Fcst')
-        for record in geometryData:
+        for record in gridData:
             self.assertLessEqual(record.getAttribute('modelName'), 'Fcst')
 
     def testGetDataWithInTuple(self):
         collection = ('Fcst', 'SAT')
-        geometryData = self._runConstraintTest('modelName', 'in', collection)
+        gridData = self._runConstraintTest('modelName', 'in', collection)
-        for record in geometryData:
+        for record in gridData:
             self.assertIn(record.getAttribute('modelName'), collection)
 
     def testGetDataWithInList(self):
         collection = ['Fcst', 'SAT']
-        geometryData = self._runConstraintTest('modelName', 'in', collection)
+        gridData = self._runConstraintTest('modelName', 'in', collection)
-        for record in geometryData:
+        for record in gridData:
             self.assertIn(record.getAttribute('modelName'), collection)
 
     def testGetDataWithInGenerator(self):
         collection = ('Fcst', 'SAT')
         generator = (item for item in collection)
-        geometryData = self._runConstraintTest('modelName', 'in', generator)
+        gridData = self._runConstraintTest('modelName', 'in', generator)
-        for record in geometryData:
+        for record in gridData:
             self.assertIn(record.getAttribute('modelName'), collection)
 
     def testGetDataWithNotInList(self):
         collection = ('Fcst', 'SAT')
-        geometryData = self._runConstraintTest('modelName', 'not in', collection)
+        gridData = self._runConstraintTest('modelName', 'not in', collection)
-        for record in geometryData:
+        for record in gridData:
             self.assertNotIn(record.getAttribute('modelName'), collection)
 
     def testGetDataWithInvalidConstraintTypeThrowsException(self):

pythonPackages/ufpy/test/dafTests/testGrid.py

@@ -21,10 +21,11 @@
 from __future__ import print_function
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 from shapely.geometry import box, Point
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -44,6 +45,9 @@ import unittest
 #    10/13/16        5942          bsteffen       Test envelopes
 #    11/08/16        5985          tgurney        Skip certain tests when no
 #                                                 data is available
+#    12/07/16        5981          tgurney        Parameterize
+#    01/06/17        5981          tgurney        Skip envelope test when no
+#                                                 data is available
 #
 
 
@@ -54,8 +58,6 @@ class GridTestCase(baseDafTestCase.DafTestCase):
 
     model = 'GFS160'
 
-    envelope = box(-97.0, 41.0, -96.0, 42.0)
-
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier('info.datasetId', self.model)
@@ -110,18 +112,18 @@ class GridTestCase(baseDafTestCase.DafTestCase):
         req.addIdentifier('info.datasetId', self.model)
         req.setLevels('2FHAG')
         req.setParameters('T')
-        req.setEnvelope(self.envelope)
+        req.setEnvelope(params.ENVELOPE)
         gridData = self.runGridDataTest(req)
-        if not gridData:
+        if len(gridData) == 0:
-            raise unittest.SkipTest('no data available')
+            raise unittest.SkipTest("No data available")
         lons, lats = gridData[0].getLatLonCoords()
         lons = lons.reshape(-1)
         lats = lats.reshape(-1)
         
         # Ensure all points are within one degree of the original box
         # to allow slight margin of error for reprojection distortion.
-        testEnv = box(self.envelope.bounds[0] - 1, self.envelope.bounds[1] - 1,
+        testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1,
-                      self.envelope.bounds[2] + 1, self.envelope.bounds[3] + 1 )
+                      params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 )
         
         for i in range(len(lons)):
             self.assertTrue(testEnv.contains(Point(lons[i], lats[i])))

pythonPackages/ufpy/test/dafTests/testHydro.py

@@ -20,8 +20,8 @@
 
 from __future__ import print_function
 import datetime
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange

pythonPackages/ufpy/test/dafTests/testLdadMesonet.py

@@ -20,7 +20,7 @@
 
 from __future__ import print_function
 from shapely.geometry import Polygon
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
 import unittest

pythonPackages/ufpy/test/dafTests/testMaps.py

@@ -20,8 +20,8 @@
 
 from __future__ import print_function
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 import baseDafTestCase
 import unittest
@@ -40,6 +40,7 @@ import unittest
 #    06/13/16        5574          mapeters       Add advanced query tests
 #    06/21/16        5548          tgurney        Skip tests that cause errors
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    01/06/17        5981          tgurney        Do not check data times
 #
 #
 
@@ -71,7 +72,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase):
         req.setLocationNames('OAX')
         req.addIdentifier('cwa', 'OAX')
         req.setParameters('countyname', 'state', 'fips')
-        self.runGeometryDataTest(req)
+        self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testRequestingTimesThrowsTimeAgnosticDataException(self):
         req = DAL.newDataRequest(self.datatype)
@@ -104,22 +105,6 @@ class MapsTestCase(baseDafTestCase.DafTestCase):
         with self.assertRaises(ThriftRequestException):
             idValues = DAL.getIdentifierValues(req, 'state')
 
-    @unittest.skip('avoid EDEX error')
-    def testGetColumnIdValuesWithNonexistentTableThrowsException(self):
-        req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('table', 'mapdata.nonexistentjunk')
-        req.addIdentifier('geomField', 'the_geom')
-        with self.assertRaises(ThriftRequestException):
-            idValues = DAL.getIdentifierValues(req, 'state')
-
-    @unittest.skip('avoid EDEX error')
-    def testGetNonexistentColumnIdValuesThrowsException(self):
-        req = DAL.newDataRequest(self.datatype)
-        req.addIdentifier('table', 'mapdata.county')
-        req.addIdentifier('geomField', 'the_geom')
-        with self.assertRaises(ThriftRequestException):
-            idValues = DAL.getIdentifierValues(req, 'nonexistentjunk')
-
     def testGetInvalidIdentifierValuesThrowsException(self):
         self.runInvalidIdValuesTest()
 
@@ -134,7 +119,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase):
         constraint = RequestConstraint.new(operator, value)
         req.addIdentifier(key, constraint)
         req.setParameters('state', 'reservoir', 'area_sq_mi')
-        return self.runGeometryDataTest(req)
+        return self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetDataWithEqualsString(self):
         geometryData = self._runConstraintTest('state', '=', 'NE')

pythonPackages/ufpy/test/dafTests/testModelSounding.py

@@ -19,10 +19,11 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -40,6 +41,9 @@ import unittest
 #    06/30/16        5725          tgurney        Add test for NOT IN
 #    11/10/16        5985          tgurney        Mark expected failures prior
 #                                                 to 17.3.1
+#    12/07/16        5981          tgurney        Parameterize
+#    12/19/16        5981          tgurney        Remove pre-17.3 expected fails
+#    12/20/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -51,31 +55,25 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
 
     def testGetAvailableParameters(self):
         req = DAL.newDataRequest(self.datatype)
-
         self.runParametersTest(req)
 
     def testGetAvailableLocations(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier("reportType", "ETA")
-
         self.runLocationsTest(req)
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier("reportType", "ETA")
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
-
         self.runTimesTest(req)
 
-    @unittest.expectedFailure
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
         req.addIdentifier("reportType", "ETA")
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
         req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
-
         print("Testing getGeometryData()")
-
         geomData = DAL.getGeometryData(req)
         print("Number of geometry records: " + str(len(geomData)))
         print("Sample geometry data:")
@@ -84,18 +82,32 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
             # One dimensional parameters are reported on the 0.0UNKNOWN level.
             # 2D parameters are reported on MB levels from pressure.
             if record.getLevel() == "0.0UNKNOWN":
-                print(" sfcPress=" + record.getString("sfcPress") + record.getUnit("sfcPress"), end="")
+                print(" sfcPress=" + record.getString("sfcPress") +
-                print(" temp2=" + record.getString("temp2") + record.getUnit("temp2"), end="")
+                      record.getUnit("sfcPress"), end="")
-                print(" q2=" + record.getString("q2") + record.getUnit("q2"), end="")
+                print(" temp2=" + record.getString("temp2") +
-
+                      record.getUnit("temp2"), end="")
+                print(" q2=" + record.getString("q2") +
+                      record.getUnit("q2"), end="")
             else:
-                print(" pressure=" + record.getString("pressure") + record.getUnit("pressure"), end="")
+                print(" pressure=" + record.getString("pressure") +
-                print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="")
+                      record.getUnit("pressure"), end="")
-                print(" specHum=" + record.getString("specHum") + record.getUnit("specHum"), end="")
+                print(" temperature=" + record.getString("temperature") +
+                      record.getUnit("temperature"), end="")
+                print(" specHum=" + record.getString("specHum") +
+                      record.getUnit("specHum"), end="")
             print(" geometry=" + str(record.getGeometry()))
-
         print("getGeometryData() complete\n\n")
 
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.addIdentifier("reportType", "ETA")
+        req.setEnvelope(params.ENVELOPE)
+        req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2")
+        print("Testing getGeometryData()")
+        data = DAL.getGeometryData(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
+
     def testGetIdentifierValues(self):
         req = DAL.newDataRequest(self.datatype)
         optionalIds = set(DAL.getOptionalIdentifiers(req))
@@ -111,7 +123,7 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
         req = DAL.newDataRequest(self.datatype)
         constraint = RequestConstraint.new(operator, value)
         req.setParameters('dataURI')
-        req.setLocationNames('KOMA', 'KORD', 'KOFK', 'KLNK')
+        req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK')
         req.addIdentifier(key, constraint)
         return self.runGeometryDataTest(req)
 
@@ -123,13 +135,11 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
     #
     # Can also eyeball the number of returned records.
 
-    @unittest.expectedFailure
     def testGetDataWithEqualsString(self):
         geometryData = self._runConstraintTest('reportType', '=', 'ETA')
         for record in geometryData:
             self.assertIn('/ETA/', record.getString('dataURI'))
 
-    @unittest.expectedFailure
     def testGetDataWithEqualsUnicode(self):
         geometryData = self._runConstraintTest('reportType', '=', u'ETA')
         for record in geometryData:
@@ -137,37 +147,29 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
 
     # No numeric tests since no numeric identifiers are available.
 
-    @unittest.expectedFailure
     def testGetDataWithEqualsNone(self):
         geometryData = self._runConstraintTest('reportType', '=', None)
 
-    @unittest.expectedFailure
     def testGetDataWithNotEquals(self):
         geometryData = self._runConstraintTest('reportType', '!=', 'ETA')
         for record in geometryData:
             self.assertNotIn('/ETA/', record.getString('dataURI'))
 
-    @unittest.expectedFailure
     def testGetDataWithNotEqualsNone(self):
         geometryData = self._runConstraintTest('reportType', '!=', None)
 
-    @unittest.expectedFailure
     def testGetDataWithGreaterThan(self):
         geometryData = self._runConstraintTest('reportType', '>', 'ETA')
 
-    @unittest.expectedFailure
     def testGetDataWithLessThan(self):
         geometryData = self._runConstraintTest('reportType', '<', 'ETA')
 
-    @unittest.expectedFailure
     def testGetDataWithGreaterThanEquals(self):
         geometryData = self._runConstraintTest('reportType', '>=', 'ETA')
 
-    @unittest.expectedFailure
     def testGetDataWithLessThanEquals(self):
         geometryData = self._runConstraintTest('reportType', '<=', 'ETA')
 
-    @unittest.expectedFailure
     def testGetDataWithInTuple(self):
         collection = ('ETA', 'GFS')
         geometryData = self._runConstraintTest('reportType', 'in', collection)
@@ -175,7 +177,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
             dataURI = record.getString('dataURI')
             self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI)
 
-    @unittest.expectedFailure
     def testGetDataWithInList(self):
         collection = ['ETA', 'GFS']
         geometryData = self._runConstraintTest('reportType', 'in', collection)
@@ -183,7 +184,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase):
             dataURI = record.getString('dataURI')
             self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI)
 
-    @unittest.expectedFailure
     def testGetDataWithInGenerator(self):
         collection = ('ETA', 'GFS')
         generator = (item for item in collection)

pythonPackages/ufpy/test/dafTests/testObs.py

@@ -19,10 +19,11 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -38,6 +39,8 @@ import unittest
 #    06/09/16        5587          bsteffen       Add getIdentifierValues tests
 #    06/13/16        5574          tgurney        Add advanced query tests
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -57,14 +60,22 @@ class ObsTestCase(baseDafTestCase.DafTestCase):
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
         self.runTimesTest(req)
 
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
         req.setParameters("temperature", "seaLevelPress", "dewpoint")
-        self.runGeometryDataTest(req)
+        data = self.runGeometryDataTest(req)
+
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setParameters("temperature", "seaLevelPress", "dewpoint")
+        data = self.runGeometryDataTest(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))
 
     def testGetIdentifierValues(self):
         req = DAL.newDataRequest(self.datatype)
@@ -81,7 +92,7 @@ class ObsTestCase(baseDafTestCase.DafTestCase):
         req = DAL.newDataRequest(self.datatype)
         constraint = RequestConstraint.new(operator, value)
         req.setParameters("temperature", "reportType")
-        req.setLocationNames("KOMA")
+        req.setLocationNames(params.OBS_STATION)
         req.addIdentifier(key, constraint)
         return self.runGeometryDataTest(req)
 

pythonPackages/ufpy/test/dafTests/testPirep.py

@@ -19,9 +19,10 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -34,6 +35,8 @@ import unittest
 #    01/19/16        4795          mapeters       Initial Creation.
 #    04/11/16        5548          tgurney        Cleanup
 #    04/18/16        5548          tgurney        More cleanup
+#    12/07/16        5981          tgurney        Parameterize
+#    12/20/16        5981          tgurney        Add envelope test
 #
 #
 
@@ -53,16 +56,14 @@ class PirepTestCase(baseDafTestCase.DafTestCase):
 
     def testGetAvailableTimes(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames('OMA')
+        req.setLocationNames(params.AIRPORT)
         self.runTimesTest(req)
 
     def testGetGeometryData(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setLocationNames('OMA')
+        req.setLocationNames(params.AIRPORT)
         req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
-
         print("Testing getGeometryData()")
-
         geomData = DAL.getGeometryData(req)
         self.assertIsNotNone(geomData)
         print("Number of geometry records: " + str(len(geomData)))
@@ -78,6 +79,13 @@ class PirepTestCase(baseDafTestCase.DafTestCase):
                 print(" hazardType=" + record.getString("hazardType"), end="")
                 print(" turbType=" + record.getString("turbType"), end="")
             print(" geometry=", record.getGeometry())
-
         print("getGeometryData() complete\n")
 
+    def testGetGeometryDataWithEnvelope(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setParameters("temperature", "windSpeed", "hazardType", "turbType")
+        req.setEnvelope(params.ENVELOPE)
+        print("Testing getGeometryData()")
+        data = DAL.getGeometryData(req)
+        for item in data:
+            self.assertTrue(params.ENVELOPE.contains(item.getGeometry()))

pythonPackages/ufpy/test/dafTests/testPracticeWarning.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
 import testWarning

pythonPackages/ufpy/test/dafTests/testProfiler.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 import baseDafTestCase
 import unittest

pythonPackages/ufpy/test/dafTests/testRadarGraphics.py

@@ -0,0 +1,95 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+import unittest
+
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+from ufpy.dataaccess import DataAccessLayer as DAL
+
+import baseRadarTestCase
+import params
+
+
+#
+# Test DAF support for radar graphics data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    08/25/16        2671          tgurney        Initial creation.
+#    08/31/16        2671          tgurney        Add mesocyclone
+#    09/08/16        2671          tgurney        Add storm track
+#    09/27/16        2671          tgurney        Add hail index
+#    09/30/16        2671          tgurney        Add TVS
+#    12/07/16        5981          tgurney        Parameterize
+#    12/19/16        5981          tgurney        Do not check data times on
+#                                                 returned data
+#
+#
+class RadarGraphicsTestCase(baseRadarTestCase.BaseRadarTestCase):
+    """Test DAF support for radar data"""
+
+    datatype = 'radar'
+
+    def runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.setParameters('166')
+        # TODO: Cannot check datatimes on the result because the times returned
+        # by getAvailableTimes have level = -1.0, while the time on the actual
+        # data has the correct level set (>= 0.0).
+        return self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataMeltingLayer(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters('166')
+        self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataMesocyclone(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters('141')
+        self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataStormTrack(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters('58')
+        self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataHailIndex(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters('59')
+        self.runGeometryDataTest(req, checkDataTimes=False)
+
+    def testGetGeometryDataTVS(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters('61')
+        self.runGeometryDataTest(req, checkDataTimes=False)

pythonPackages/ufpy/test/dafTests/testRadarGrid.py

@@ -0,0 +1,61 @@
+##
+# This software was developed and / or modified by Raytheon Company,
+# pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+#
+# U.S. EXPORT CONTROLLED TECHNICAL DATA
+# This software product contains export-restricted data whose
+# export/transfer/disclosure is restricted by U.S. law. Dissemination
+# to non-U.S. persons whether in the United States or abroad requires
+# an export license or other authorization.
+#
+# Contractor Name:        Raytheon Company
+# Contractor Address:     6825 Pine Street, Suite 340
+#                         Mail Stop B8
+#                         Omaha, NE 68106
+#                         402.291.0100
+#
+# See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+# further licensing information.
+##
+
+from ufpy.dataaccess import DataAccessLayer as DAL
+from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
+
+import baseRadarTestCase
+import params
+import unittest
+
+#
+# Test DAF support for radar grid data
+#
+#     SOFTWARE HISTORY
+#
+#    Date            Ticket#       Engineer       Description
+#    ------------    ----------    -----------    --------------------------
+#    08/25/16        2671          tgurney        Initial creation
+#
+#
+
+
+class RadarTestCase(baseRadarTestCase.BaseRadarTestCase):
+    """Test DAF support for radar data"""
+
+    datatype = 'radar'
+
+    parameterList = ['94']
+
+    def runConstraintTest(self, key, operator, value):
+        req = DAL.newDataRequest(self.datatype)
+        constraint = RequestConstraint.new(operator, value)
+        req.addIdentifier(key, constraint)
+        req.setParameters(*self.parameterList)
+        # Don't test shapes since they may differ.
+        return self.runGridDataTest(req, testSameShape=False)
+
+    def testGetGridData(self):
+        req = DAL.newDataRequest(self.datatype)
+        req.setEnvelope(params.ENVELOPE)
+        req.setLocationNames(self.radarLoc)
+        req.setParameters(*self.parameterList)
+        # Don't test shapes since they may differ.
+        self.runGridDataTest(req, testSameShape=False)

pythonPackages/ufpy/test/dafTests/testRadarSpatial.py

@@ -20,10 +20,11 @@
 
 from __future__ import print_function
 from shapely.geometry import box
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase
+import params
 import unittest
 
 #
@@ -41,6 +42,8 @@ import unittest
 #                                                 superclass
 #    06/13/16        5574          tgurney        Add advanced query tests
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    12/07/16        5981          tgurney        Parameterize
+#    01/06/17        5981          tgurney        Do not check data times
 #
 #
 
@@ -50,14 +53,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
 
     datatype = "radar_spatial"
 
-    envelope = box(-97.0, 41.0, -96.0, 42.0)
-    """
-    Default request area (box around KOAX)
-    """
-
     def testGetAvailableLocations(self):
         req = DAL.newDataRequest(self.datatype)
-        req.setEnvelope(self.envelope)
+        req.setEnvelope(params.ENVELOPE)
         self.runLocationsTest(req)
 
     def testGetAvailableParameters(self):
@@ -71,7 +69,7 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
         req = DAL.newDataRequest(self.datatype)
         req.setLocationNames("TORD", "TMDW")
         req.setParameters("wfo_id", "name", "elevmeter")
-        self.runGeometryDataTest(req)
+        self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testRequestingTimesThrowsTimeAgnosticDataException(self):
         req = DAL.newDataRequest(self.datatype)
@@ -82,17 +80,17 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
         constraint = RequestConstraint.new(operator, value)
         req.addIdentifier(key, constraint)
         req.setParameters('elevmeter', 'eqp_elv', 'wfo_id', 'immutablex')
-        return self.runGeometryDataTest(req)
+        return self.runGeometryDataTest(req, checkDataTimes=False)
 
     def testGetDataWithEqualsString(self):
-        geometryData = self._runConstraintTest('wfo_id', '=', 'OAX')
+        geometryData = self._runConstraintTest('wfo_id', '=', params.SITE_ID)
         for record in geometryData:
-            self.assertEqual(record.getString('wfo_id'), 'OAX')
+            self.assertEqual(record.getString('wfo_id'), params.SITE_ID)
 
     def testGetDataWithEqualsUnicode(self):
-        geometryData = self._runConstraintTest('wfo_id', '=', u'OAX')
+        geometryData = self._runConstraintTest('wfo_id', '=', unicode(params.SITE_ID))
         for record in geometryData:
-            self.assertEqual(record.getString('wfo_id'), 'OAX')
+            self.assertEqual(record.getString('wfo_id'), params.SITE_ID)
 
     def testGetDataWithEqualsInt(self):
         geometryData = self._runConstraintTest('immutablex', '=', 57)
@@ -115,9 +113,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
             self.assertEqual(record.getType('wfo_id'), 'NULL')
 
     def testGetDataWithNotEquals(self):
-        geometryData = self._runConstraintTest('wfo_id', '!=', 'OAX')
+        geometryData = self._runConstraintTest('wfo_id', '!=', params.SITE_ID)
         for record in geometryData:
-            self.assertNotEquals(record.getString('wfo_id'), 'OAX')
+            self.assertNotEquals(record.getString('wfo_id'), params.SITE_ID)
 
     def testGetDataWithNotEqualsNone(self):
         geometryData = self._runConstraintTest('wfo_id', '!=', None)
@@ -145,33 +143,33 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase):
             self.assertLessEqual(record.getNumber('eqp_elv'), 138)
 
     def testGetDataWithInTuple(self):
-        collection = ('OAX', 'GID')
+        collection = (params.SITE_ID, 'GID')
         geometryData = self._runConstraintTest('wfo_id', 'in', collection)
         for record in geometryData:
             self.assertIn(record.getString('wfo_id'), collection)
 
     def testGetDataWithInList(self):
-        collection = ['OAX', 'GID']
+        collection = [params.SITE_ID, 'GID']
         geometryData = self._runConstraintTest('wfo_id', 'in', collection)
         for record in geometryData:
             self.assertIn(record.getString('wfo_id'), collection)
 
     def testGetDataWithInGenerator(self):
-        collection = ('OAX', 'GID')
+        collection = (params.SITE_ID, 'GID')
         generator = (item for item in collection)
         geometryData = self._runConstraintTest('wfo_id', 'in', generator)
         for record in geometryData:
             self.assertIn(record.getString('wfo_id'), collection)
 
     def testGetDataWithNotInList(self):
-        collection = ['OAX', 'GID']
+        collection = [params.SITE_ID, 'GID']
         geometryData = self._runConstraintTest('wfo_id', 'not in', collection)
         for record in geometryData:
             self.assertNotIn(record.getString('wfo_id'), collection)
 
     def testGetDataWithInvalidConstraintTypeThrowsException(self):
         with self.assertRaises(ValueError):
-            self._runConstraintTest('wfo_id', 'junk', 'OAX')
+            self._runConstraintTest('wfo_id', 'junk', params.SITE_ID)
 
     def testGetDataWithInvalidConstraintValueThrowsException(self):
         with self.assertRaises(TypeError):

pythonPackages/ufpy/test/dafTests/testSatellite.py

@@ -20,7 +20,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 
 import baseDafTestCase

pythonPackages/ufpy/test/dafTests/testSfcObs.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase

pythonPackages/ufpy/test/dafTests/testTopo.py

@@ -19,8 +19,8 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
-from awips.ThriftClient import ThriftRequestException
+from ufpy.ThriftClient import ThriftRequestException
 
 import baseDafTestCase
 import shapely.geometry
@@ -39,7 +39,7 @@ import unittest
 #    05/26/16        5587          tgurney        Add test for
 #                                                 getIdentifierValues()
 #    06/01/16        5587          tgurney        Update testGetIdentifierValues
-#
+#    07/18/17        6253          randerso       Removed referenced to GMTED
 #
 
 
@@ -61,7 +61,7 @@ class TopoTestCase(baseDafTestCase.DafTestCase):
         print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n")
         print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n")
 
-        for topoFile in ["gmted2010", "gtopo30"]:
+        for topoFile in ["gtopo30"]:
             print("\n" + topoFile)
             req.addIdentifier("topoFile", topoFile)
             gridData = DAL.getGridData(req)

pythonPackages/ufpy/test/dafTests/testWarning.py

@@ -19,7 +19,7 @@
 ##
 
 from __future__ import print_function
-from awips.dataaccess import DataAccessLayer as DAL
+from ufpy.dataaccess import DataAccessLayer as DAL
 
 from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint
 import baseDafTestCase
@@ -42,6 +42,7 @@ import unittest
 #    06/13/16        5574          tgurney        Fix checks for None
 #    06/21/16        5548          tgurney        Skip tests that cause errors
 #    06/30/16        5725          tgurney        Add test for NOT IN
+#    12/12/16        5981          tgurney        Improve test performance
 #
 #
 
@@ -81,12 +82,10 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
         self.runGeometryDataTest(req)
 
     def testFilterOnLocationName(self):
-        allRecordsCount = len(self._getAllRecords())
         allLocationNames = self._getLocationNames()
-        if allRecordsCount == 0:
+        if len(allLocationNames) == 0:
             errmsg = "No {0} data exists on {1}. Try again with {0} data."
             raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST))
-        if len(allLocationNames) != 1:
         testCount = 3  # number of different location names to test
         for locationName in allLocationNames[:testCount]:
             req = DAL.newDataRequest()
@@ -94,7 +93,6 @@ class WarningTestCase(baseDafTestCase.DafTestCase):
             req.setParameters('id')
             req.setLocationNames(locationName)
             geomData = DAL.getGeometryData(req)
-                self.assertLess(len(geomData), allRecordsCount)
             for geom in geomData:
                 self.assertEqual(geom.getLocationName(), locationName)
 

pythonPackages/ufpy/test/testQpidTimeToLive.py

@@ -52,7 +52,7 @@ class ListenThread(threading.Thread):
         threading.Thread.__init__(self)
     
     def run(self):
-        from awips import QpidSubscriber
+        from ufpy import QpidSubscriber
         self.qs = QpidSubscriber.QpidSubscriber(self.hostname, self.portNumber, True)        
         self.qs.topicSubscribe(self.topicName, self.receivedMessage)