From 7dcbefda6169220ab4da0a42faeafe17251dc138 Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Sat, 30 Sep 2017 22:41:17 -0600 Subject: [PATCH] dataaccess checkout from 17.3.1 --- .../dataaccess/BinLightningAccessFactory.java | 81 +- .../gfe/dataaccess/GFEDataAccessUtil.java | 26 +- .../gfe/dataaccess/GFEGridFactory.java | 23 +- .../dataaccess/GridDataAccessFactory.java | 124 +-- .../dataaccess/RadarDataAccessFactory.java | 852 ++++++++++++++++++ .../dataaccess/RadarGeometryDataUtil.java | 563 ++++++++++++ .../dataplugin/radar/level3/SCLPacket.java | 55 ++ .../radar/util/RadarRecordUtil.java | 301 +++++-- .../radar/util/RadarTextProductUtil.java | 3 +- .../dataaccess/SatelliteGridFactory.java | 15 +- .../META-INF/MANIFEST.MF | 2 +- .../res/spring/dataaccess-request.xml | 69 +- .../handler/GetGridDataHandler.java | 14 +- .../handler/GetGridLatLonHandler.java | 70 ++ .../base/python/dataaccess/JData.py | 7 +- .../base/python/dataaccess/JDataRequest.py | 7 +- .../base/python/dataaccess/JGeometryData.py | 7 +- .../base/python/dataaccess/JGridData.py | 12 +- .../base/python/dataaccess/JepRouter.py | 8 +- .../base/python/dataaccess/__init__.py | 9 +- .../ThriftSerializationContext.py | 295 +++--- .../dynamicserialize/adapters/__init__.py | 117 ++- .../com/raytheon/uf/common/__init__.py | 17 +- .../raytheon/uf/common/dataplugin/__init__.py | 8 +- pythonPackages/ufpy/NotificationMessage.py | 2 +- pythonPackages/ufpy/QpidSubscriber.py | 20 +- pythonPackages/ufpy/UsageArgumentParser.py | 29 +- pythonPackages/ufpy/__init__.py | 2 +- .../ufpy/dataaccess/CombinedTimeQuery.py | 2 +- .../ufpy/dataaccess/DataAccessLayer.py | 29 +- .../ufpy/dataaccess/DataNotificationLayer.py | 14 +- pythonPackages/ufpy/dataaccess/DataQueue.py | 2 +- pythonPackages/ufpy/dataaccess/PyData.py | 2 +- .../ufpy/dataaccess/PyGeometryData.py | 4 +- .../ufpy/dataaccess/PyGeometryNotification.py | 2 +- pythonPackages/ufpy/dataaccess/PyGridData.py | 14 +- .../ufpy/dataaccess/PyGridNotification.py | 2 +- .../ufpy/dataaccess/PyNotification.py | 8 +- .../ufpy/dataaccess/SoundingsSupport.py | 4 +- .../ufpy/dataaccess/ThriftClientRouter.py | 76 +- pythonPackages/ufpy/dataaccess/__init__.py | 2 +- pythonPackages/ufpy/gfe/IFPClient.py | 2 +- pythonPackages/ufpy/gfe/__init__.py | 2 +- pythonPackages/ufpy/qpidingest.py | 26 +- pythonPackages/ufpy/test/Test | 2 +- pythonPackages/ufpy/test/__init__.py | 2 +- pythonPackages/ufpy/test/dafTests/__init__.py | 2 +- .../ufpy/test/dafTests/baseBufrMosTestCase.py | 26 +- .../ufpy/test/dafTests/baseDafTestCase.py | 15 +- .../ufpy/test/dafTests/baseRadarTestCase.py | 194 ++++ pythonPackages/ufpy/test/dafTests/params.py | 43 + .../ufpy/test/dafTests/testAcars.py | 2 +- .../ufpy/test/dafTests/testAirep.py | 2 +- .../ufpy/test/dafTests/testBinLightning.py | 4 +- .../ufpy/test/dafTests/testBufrMosHpc.py | 14 +- .../ufpy/test/dafTests/testBufrMosMrf.py | 14 +- .../ufpy/test/dafTests/testBufrUa.py | 17 +- .../ufpy/test/dafTests/testClimate.py | 46 +- .../test/dafTests/testCombinedTimeQuery.py | 8 +- .../test/dafTests/testCommonObsSpatial.py | 22 +- pythonPackages/ufpy/test/dafTests/testFfmp.py | 56 +- pythonPackages/ufpy/test/dafTests/testGfe.py | 98 +- pythonPackages/ufpy/test/dafTests/testGrid.py | 22 +- .../ufpy/test/dafTests/testHydro.py | 4 +- .../ufpy/test/dafTests/testLdadMesonet.py | 2 +- pythonPackages/ufpy/test/dafTests/testMaps.py | 25 +- .../ufpy/test/dafTests/testModelSounding.py | 60 +- pythonPackages/ufpy/test/dafTests/testObs.py | 21 +- .../ufpy/test/dafTests/testPirep.py | 20 +- .../ufpy/test/dafTests/testPracticeWarning.py | 2 +- .../ufpy/test/dafTests/testProfiler.py | 2 +- .../ufpy/test/dafTests/testRadarGraphics.py | 95 ++ .../ufpy/test/dafTests/testRadarGrid.py | 61 ++ .../ufpy/test/dafTests/testRadarSpatial.py | 38 +- .../ufpy/test/dafTests/testSatellite.py | 2 +- .../ufpy/test/dafTests/testSfcObs.py | 2 +- pythonPackages/ufpy/test/dafTests/testTopo.py | 8 +- .../ufpy/test/dafTests/testWarning.py | 26 +- .../ufpy/test/testQpidTimeToLive.py | 2 +- 79 files changed, 3074 insertions(+), 814 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarDataAccessFactory.java create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarGeometryDataUtil.java create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/level3/SCLPacket.java create mode 100644 edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridLatLonHandler.java create mode 100644 pythonPackages/ufpy/test/dafTests/baseRadarTestCase.py create mode 100644 pythonPackages/ufpy/test/dafTests/params.py create mode 100644 pythonPackages/ufpy/test/dafTests/testRadarGraphics.py create mode 100644 pythonPackages/ufpy/test/dafTests/testRadarGrid.py diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightningAccessFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightningAccessFactory.java index f5f079cf6e..0f43e24d88 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightningAccessFactory.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.binlightning/src/com/raytheon/uf/common/dataplugin/binlightning/dataaccess/BinLightningAccessFactory.java @@ -1,19 +1,19 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ @@ -23,6 +23,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -30,15 +31,16 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import com.raytheon.uf.common.dataaccess.IDataRequest; import com.raytheon.uf.common.dataaccess.exception.IncompatibleRequestException; +import com.raytheon.uf.common.dataaccess.exception.TimeAgnosticDataException; import com.raytheon.uf.common.dataaccess.geom.IGeometryData; import com.raytheon.uf.common.dataaccess.geom.IGeometryData.Type; import com.raytheon.uf.common.dataaccess.impl.AbstractDataPluginFactory; import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData; import com.raytheon.uf.common.dataaccess.util.PDOUtil; -import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.binlightning.BinLightningRecord; import com.raytheon.uf.common.dataplugin.binlightning.LightningConstants; import com.raytheon.uf.common.dataquery.requests.RequestConstraint; @@ -54,6 +56,7 @@ import com.raytheon.uf.common.datastorage.records.IntegerDataRecord; import com.raytheon.uf.common.datastorage.records.LongDataRecord; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.time.BinOffset; import com.raytheon.uf.common.time.DataTime; import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Envelope; @@ -61,15 +64,15 @@ import com.vividsolutions.jts.geom.GeometryFactory; /** * Data access framework factory for bin lightning - * + * * Envelopes requests cannot be handled efficiently using metadata so all data * is retrieved and filtered within the factory. For very large requests this * may result in suboptimal performance. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
- * 
+ *
  * Date          Ticket#  Engineer  Description
  * ------------- -------- --------- --------------------------------------------
  * Jan 21, 2014  2667     bclement  Initial creation
@@ -83,9 +86,12 @@ import com.vividsolutions.jts.geom.GeometryFactory;
  * Jun 07, 2016  5587     tgurney   Change get*Identifiers() to take
  *                                  IDataRequest
  * Aug 01, 2016  2416     tgurney   Add dataURI as optional identifier
- * 
+ * Nov 30, 2016  6018     tgurney   Filter out keep-alive records from available
+ *                                  times
+ * Mar 06, 2017  6142     bsteffen  Remove dataURI as optional identifier
+ *
  *
- * + * * @author bclement */ public class BinLightningAccessFactory extends AbstractDataPluginFactory { @@ -105,7 +111,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { private static final String[] requiredKeys = { timeKey, latKey, lonKey }; - public static final String[] AVAILABLE_PARAMETERS = { + protected static final String[] AVAILABLE_PARAMETERS = { LightningConstants.INTENSITY_DATASET, LightningConstants.MSG_TYPE_DATASET, LightningConstants.STRIKE_TYPE_DATASET, @@ -116,8 +122,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { @Override public String[] getAvailableLocationNames(IDataRequest request) { - throw new IncompatibleRequestException(this.getClass() - + " does not support location names"); + throw new IncompatibleRequestException( + this.getClass() + " does not support location names"); } @Override @@ -130,11 +136,6 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { return new String[] { sourceKey }; } - @Override - public String[] getOptionalIdentifiers(IDataRequest request) { - return new String[] { PluginDataObject.DATAURI_ID }; - } - @Override protected Map buildConstraintsFromRequest( IDataRequest request) { @@ -158,7 +159,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { @Override protected IGeometryData[] getGeometryData(IDataRequest request, DbQueryResponse dbQueryResponse) { - Map> results = unpackResults(dbQueryResponse); + Map> results = unpackResults( + dbQueryResponse); List rval = new ArrayList<>(); for (Entry> resultEntry : results @@ -177,7 +179,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { /** * Add geometry data elements to dataList from data store - * + * * @param dataList * target result list * @param ds @@ -224,9 +226,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { .getFloatData(); for (int i = 0; i < timeData.length; i++) { - if (envelope != null - && !envelope.contains(longitudeData[i], - latitudeData[i])) { + if (envelope != null && !envelope.contains(longitudeData[i], + latitudeData[i])) { /* Skip any data the user doesn't want */ continue; } @@ -234,8 +235,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { DefaultGeometryData data = new DefaultGeometryData(); data.setDataTime(dt); data.addAttribute(sourceKey, source); - data.setGeometry(geomFactory.createPoint(new Coordinate( - longitudeData[i], latitudeData[i]))); + data.setGeometry(geomFactory.createPoint( + new Coordinate(longitudeData[i], latitudeData[i]))); // add the optional parameter records addParameterData(data, recordMap, k, i); dataList.add(data); @@ -251,7 +252,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { /** * Add parameters from record map to data - * + * * @param data * target geometry data * @param recordMap @@ -268,7 +269,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { String parameterName = entry.getKey(); IDataRecord record = entry.getValue().get(recordIndex); if (record instanceof IntegerDataRecord) { - int value = ((IntegerDataRecord) record).getIntData()[valueIndex]; + int value = ((IntegerDataRecord) record) + .getIntData()[valueIndex]; data.addData(parameterName, value, Type.INT); } else if (record instanceof ByteDataRecord) { int value = ((ByteDataRecord) record).getByteData()[valueIndex]; @@ -285,7 +287,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { /** * Return mapping of lightning data source to list of datasets - * + * * @param recList * @return */ @@ -311,7 +313,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { /** * Get a list of HDF5 datasets to request - * + * * @param request * @return */ @@ -323,8 +325,8 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { if (availableParams.contains(param)) { included.add(param); } else { - throw new IncompatibleRequestException(param - + " is not a valid parameter for this request"); + throw new IncompatibleRequestException( + param + " is not a valid parameter for this request"); } } return new ArrayList<>(included); @@ -332,7 +334,7 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { /** * Unpack records from response and group by HDF5 file - * + * * @param dbQueryResponse * @return */ @@ -365,4 +367,19 @@ public class BinLightningAccessFactory extends AbstractDataPluginFactory { return getAvailableValues(request, identifierKey, String.class); } + private boolean isKeepAlive(DataTime t) { + Calendar cal = t.getRefTimeAsCalendar(); + return t.getValidPeriod().getStart().equals(t.getValidPeriod().getEnd()) + && cal.get(Calendar.SECOND) == 0 + && cal.get(Calendar.MILLISECOND) == 0; + } + + @Override + public DataTime[] getAvailableTimes(IDataRequest request, + BinOffset binOffset) throws TimeAgnosticDataException { + DataTime[] times = super.getAvailableTimes(request, binOffset); + return Arrays.stream(times).filter(t -> !isKeepAlive(t)) + .collect(Collectors.toList()).toArray(new DataTime[0]); + } + } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java index a7cb1b06a1..ab3f61deb9 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEDataAccessUtil.java @@ -1,19 +1,19 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ @@ -33,13 +33,13 @@ import com.raytheon.uf.common.dataplugin.gfe.slice.IGridSlice; import com.raytheon.uf.common.serialization.comm.RequestRouter; /** - * + * * Some utility methods for querying and retrieving GFE data. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
- * 
+ *
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Mar 9, 2011            bsteffen     Initial creation
@@ -47,11 +47,11 @@ import com.raytheon.uf.common.serialization.comm.RequestRouter;
  *                                     Browser, Volume Browser, and Data Access
  *                                     Framework.
  * Jul 01, 2014 3149      randerso     Changed to use updated GetGridRequest
- * 
+ * Dec 15, 2016 6040      tgurney      Added DB_TYPE constant
+ *
  *
- * + * * @author bsteffen - * @version 1.0 */ public class GFEDataAccessUtil { @@ -71,9 +71,11 @@ public class GFEDataAccessUtil { public static final String PARM_LEVEL = PARM_ID + ".parmLevel"; + public static final String DB_TYPE = DB_ID + ".dbType"; + /** * Retrieve the GridParmInfo for a ParmID - * + * * @param parmId * @return * @throws Exception @@ -91,7 +93,7 @@ public class GFEDataAccessUtil { /** * Send a GetGridDataRequest through the requestRouter to grab a single * slice of grid data. - * + * * @param gfeRecord * @return * @throws Exception diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEGridFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEGridFactory.java index a8eb2b05d9..1bf3655952 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEGridFactory.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/dataaccess/GFEGridFactory.java @@ -62,11 +62,11 @@ import com.raytheon.uf.common.util.StringUtil; /** * A data factory for getting gfe data from the metadata database. There are * currently not any required identifiers. - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
- * 
+ *
  * Date          Ticket#  Engineer  Description
  * ------------- -------- --------- --------------------------------------------
  * Feb 04, 2013           bsteffen  Initial creation
@@ -88,9 +88,12 @@ import com.raytheon.uf.common.util.StringUtil;
  *                                  IDataRequest
  * Jun 13, 2016  5574     mapeters  Add advanced query support
  * Aug 01, 2016  2416     tgurney   Add dataURI as optional identifier
+ * Dec 15, 2016  6040     tgurney   Add dbType as optional identifier
+ * Mar 06, 2017  6142     bsteffen  Remove dataURI as optional identifier
  * 
+ *
  *
- * + * * @author bsteffen */ @@ -109,7 +112,7 @@ public class GFEGridFactory extends AbstractGridDataPluginFactory { private static final String[] OPTIONAL_IDENTIFIERS = { GFEDataAccessUtil.MODEL_NAME, GFEDataAccessUtil.MODEL_TIME, GFEDataAccessUtil.SITE_ID, MODEL_NAME, MODEL_TIME, SITE_ID, - PluginDataObject.DATAURI_ID }; + GFEDataAccessUtil.DB_TYPE }; @Override public String[] getOptionalIdentifiers(IDataRequest request) { @@ -240,7 +243,7 @@ public class GFEGridFactory extends AbstractGridDataPluginFactory { * Estimates the subgrid memory size using the grid geometry's size because * {@link #getDataSource(PluginDataObject, SubGridGeometryCalculator)} uses * an {@link OffsetDataSource} that holds the full grid data in memory. - * + * * @param gridGeom * @param subGrid * @return @@ -339,10 +342,10 @@ public class GFEGridFactory extends AbstractGridDataPluginFactory { @Override public String[] getIdentifierValues(IDataRequest request, String identifierKey) { - if (!Arrays.asList(getRequiredIdentifiers(request)).contains( - identifierKey) - && !Arrays.asList(getOptionalIdentifiers(request)).contains( - identifierKey)) { + if (!Arrays.asList(getRequiredIdentifiers(request)) + .contains(identifierKey) + && !Arrays.asList(getOptionalIdentifiers(request)) + .contains(identifierKey)) { throw new InvalidIdentifiersException(request.getDatatype(), null, Arrays.asList(new String[] { identifierKey })); } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.grid/src/com/raytheon/uf/common/dataplugin/grid/dataaccess/GridDataAccessFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.grid/src/com/raytheon/uf/common/dataplugin/grid/dataaccess/GridDataAccessFactory.java index c296fe2d1e..538fb47f30 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.grid/src/com/raytheon/uf/common/dataplugin/grid/dataaccess/GridDataAccessFactory.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.grid/src/com/raytheon/uf/common/dataplugin/grid/dataaccess/GridDataAccessFactory.java @@ -105,6 +105,7 @@ import com.vividsolutions.jts.geom.GeometryFactory; * IDataRequest * Jul 06, 2016 5728 mapeters Add advanced query support * Aug 01, 2016 2416 tgurney Add dataURI as optional identifier + * Mar 06, 2017 6142 bsteffen Remove dataURI as optional identifier * * * @@ -114,14 +115,18 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { public static final String NAMESPACE = "namespace"; - public static final String[] VALID_IDENTIFIERS = { + protected static final String[] VALID_IDENTIFIERS = { GridConstants.DATASET_ID, GridConstants.SECONDARY_ID, GridConstants.ENSEMBLE_ID, NAMESPACE, GridConstants.MASTER_LEVEL_NAME, GridConstants.LEVEL_ONE, - GridConstants.LEVEL_TWO, PluginDataObject.DATAURI_ID }; + GridConstants.LEVEL_TWO }; @Override public String[] getOptionalIdentifiers(IDataRequest request) { + return getGridOptionalIdentifiers(); + } + + public static String[] getGridOptionalIdentifiers() { return VALID_IDENTIFIERS; } @@ -140,8 +145,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { DAFGridQueryAssembler assembler = new DAFGridQueryAssembler(); if (identifiers != null && identifiers.containsKey(NAMESPACE)) { - assembler.setNamespace(getStringIdentifier(identifiers, - NAMESPACE)); + assembler.setNamespace( + getStringIdentifier(identifiers, NAMESPACE)); } if (request.getParameters() != null) { @@ -157,10 +162,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { checkForLevelConflict(request.getLevels(), identifiers); for (Level level : request.getLevels()) { - assembler.setMasterLevelName(level.getMasterLevel() - .getName()); - assembler.setLevelUnits(level.getMasterLevel() - .getUnitString()); + assembler.setMasterLevelName( + level.getMasterLevel().getName()); + assembler.setLevelUnits( + level.getMasterLevel().getUnitString()); assembler.setLevelOneValue(level.getLevelonevalue()); assembler.setLevelTwoValue(level.getLeveltwovalue()); // TODO Theoretically merging these could end badly if there @@ -200,9 +205,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { identifiers, GridConstants.SECONDARY_ID)); } if (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME)) { - assembler - .setMasterLevelNameConstraint(getIdentifierConstraint( - identifiers, + assembler.setMasterLevelNameConstraint( + getIdentifierConstraint(identifiers, GridConstants.MASTER_LEVEL_NAME)); } if (identifiers.containsKey(GridConstants.LEVEL_ONE)) { @@ -256,7 +260,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { return (String) value; } else { throw new IncompatibleRequestException( - "Only string identifier values are valid for '" + key + "'"); + "Only string identifier values are valid for '" + key + + "'"); } } @@ -275,11 +280,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { */ private static void checkForLevelConflict(Level[] levels, Map identifiers) { - if (levels.length > 0 - && identifiers != null + if (levels.length > 0 && identifiers != null && (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME) - || identifiers.containsKey(GridConstants.LEVEL_ONE) || identifiers - .containsKey(GridConstants.LEVEL_TWO))) { + || identifiers.containsKey(GridConstants.LEVEL_ONE) + || identifiers.containsKey(GridConstants.LEVEL_TWO))) { throw new DataRetrievalException( "Conflict between the request levels and request " + "identifiers. Please set the levels either as" @@ -300,8 +304,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { for (Entry sourceEntry : source.entrySet()) { String key = sourceEntry.getKey(); RequestConstraint sourceConstraint = sourceEntry.getValue(); - RequestConstraint targetConstraint = target.get(sourceEntry - .getKey()); + RequestConstraint targetConstraint = target + .get(sourceEntry.getKey()); if (targetConstraint == null) { target.put(key, sourceConstraint); } else if (!sourceConstraint.equals(targetConstraint)) { @@ -309,8 +313,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { // TODO we don't necessarily want to always add. This could // result in something like IN MB,FHAG,MB,MB,MB, but we also // don't want to parse the in list all the time. - targetConstraint.addToConstraintValueList(sourceConstraint - .getConstraintValue()); + targetConstraint.addToConstraintValueList( + sourceConstraint.getConstraintValue()); } } } @@ -319,13 +323,13 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { protected IGridData constructGridDataResponse(IDataRequest request, PluginDataObject pdo, GridGeometry2D gridGeometry, DataSource dataSource) { - if (pdo instanceof GridRecord == false) { - throw new DataRetrievalException(this.getClass().getSimpleName() - + " cannot handle " + pdo.getClass().getSimpleName()); + if (pdo instanceof GridRecord) { + GridRecord gridRecord = (GridRecord) pdo; + return constructGridDataResponse(request, gridRecord, gridGeometry, + dataSource); } - GridRecord gridRecord = (GridRecord) pdo; - return constructGridDataResponse(request, gridRecord, gridGeometry, - dataSource); + throw new DataRetrievalException(this.getClass().getSimpleName() + + " cannot handle " + pdo.getClass().getSimpleName()); } public static IGridData constructGridDataResponse(IDataRequest request, @@ -341,17 +345,16 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { // perform reverse mappings so the parameters and levels that are // returned match exactly what was requested. String namespace = getStringIdentifier(identifiers, NAMESPACE); - List requestParameters = Arrays.asList(request - .getParameters()); + List requestParameters = Arrays + .asList(request.getParameters()); parameter = reverseResolveMapping(ParameterMapper.getInstance(), parameter, namespace, requestParameters); if (identifiers.containsKey(GridConstants.DATASET_ID)) { RequestConstraint requestedDatasetsConstraint = getIdentifierConstraint( identifiers, GridConstants.DATASET_ID); - datasetId = reverseResolveMapping( - DatasetIdMapper.getInstance(), datasetId, namespace, - requestedDatasetsConstraint); + datasetId = reverseResolveMapping(DatasetIdMapper.getInstance(), + datasetId, namespace, requestedDatasetsConstraint); } if (identifiers.containsKey(GridConstants.MASTER_LEVEL_NAME)) { RequestConstraint requestedMasterLevelConstraint = getIdentifierConstraint( @@ -448,8 +451,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { */ @Override public String[] getAvailableParameters(IDataRequest request) { - return getAvailableValues(request, - GridConstants.PARAMETER_ABBREVIATION, String.class); + return getAvailableValues(request, GridConstants.PARAMETER_ABBREVIATION, + String.class); } /** @@ -478,8 +481,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { dbQueryResponse.getNumResults()); for (Entry> entry : sortedRecords .entrySet()) { - result.addAll(getGeometryData(request, entry.getKey(), - entry.getValue())); + result.addAll( + getGeometryData(request, entry.getKey(), entry.getValue())); } return result.toArray(new IGeometryData[0]); @@ -519,10 +522,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { DefaultGeometryData data = key.toGeometryData(); DirectPosition2D llPoint = findResponsePoint(key.getGridGeometry(), point.x, point.y); - data.setGeometry(new GeometryFactory().createPoint(new Coordinate( - llPoint.x, llPoint.y))); - data.setLocationName(data.getLocationName() + "-" + point.x + "," - + point.y); + data.setGeometry(new GeometryFactory() + .createPoint(new Coordinate(llPoint.x, llPoint.y))); + data.setLocationName( + data.getLocationName() + "-" + point.x + "," + point.y); Request request = Request.buildPointRequest(point); populateGeometryData(records, request, new DefaultGeometryData[] { data }); @@ -542,15 +545,17 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { .getWidth() * gridRange.getHeight())]; GeometryFactory geometryFactory = new GeometryFactory(); int index = 0; - for (int y = (int) gridRange.getMinY(); y < gridRange.getMaxY(); y += 1) { - for (int x = (int) gridRange.getMinX(); x < gridRange.getMaxX(); x += 1) { + for (int y = (int) gridRange.getMinY(); y < gridRange + .getMaxY(); y += 1) { + for (int x = (int) gridRange.getMinX(); x < gridRange + .getMaxX(); x += 1) { data[index] = key.toGeometryData(); DirectPosition2D llPoint = findResponsePoint( key.getGridGeometry(), x, y); data[index].setGeometry(geometryFactory .createPoint(new Coordinate(llPoint.x, llPoint.y))); - data[index].setLocationName(data[index].getLocationName() + "-" - + x + "," + y); + data[index].setLocationName( + data[index].getLocationName() + "-" + x + "," + y); index += 1; } } @@ -563,10 +568,10 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { @Override public String[] getIdentifierValues(IDataRequest request, String identifierKey) { - if (!Arrays.asList(getRequiredIdentifiers(request)).contains( - identifierKey) - && !Arrays.asList(getOptionalIdentifiers(request)).contains( - identifierKey)) { + if (!Arrays.asList(getRequiredIdentifiers(request)) + .contains(identifierKey) + && !Arrays.asList(getOptionalIdentifiers(request)) + .contains(identifierKey)) { throw new InvalidIdentifiersException(request.getDatatype(), null, Arrays.asList(identifierKey)); } @@ -602,25 +607,26 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { .getFloatData(); if (rawArray.length != data.length) { throw new DataRetrievalException( - "Unexpected response of size " - + rawArray.length + "Unexpected response of size " + rawArray.length + " when expected size is " - + data.length + " for record " + record); + + data.length + " for record " + + record); } for (int i = 0; i < data.length; i += 1) { data[i].addData(parameter.getAbbreviation(), rawArray[i], parameter.getUnit()); } } else { - String type = dataRecord == null ? "null" : dataRecord - .getClass().getSimpleName(); + String type = dataRecord == null ? "null" + : dataRecord.getClass().getSimpleName(); throw new DataRetrievalException("Unexpected record type(" + type + ") for " + record); } } catch (Exception e) { throw new DataRetrievalException( "Failed to retrieve the IDataRecord for GridRecord: " - + record.toString(), e); + + record.toString(), + e); } } } @@ -669,7 +675,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { int maxX = (int) Math.round(testPoints[2]); int maxY = (int) Math.round(testPoints[3]); GridEnvelope2D gridRange = gridGeometry.getGridRange2D(); - if (minX == maxX && minY == maxY && gridRange.contains(minX, minY)) { + if (minX == maxX && minY == maxY + && gridRange.contains(minX, minY)) { return new Point(minX, minY); } else { return null; @@ -696,8 +703,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { private final int hashCode; - public GridGeometryKey(Level level, DataTime dataTime, - String datasetId, GridGeometry2D gridGeometry) { + public GridGeometryKey(Level level, DataTime dataTime, String datasetId, + GridGeometry2D gridGeometry) { this.level = level; this.dataTime = dataTime; this.datasetId = datasetId; @@ -716,9 +723,8 @@ public class GridDataAccessFactory extends AbstractGridDataPluginFactory { } public GridGeometryKey(GridRecord record) { - this(record.getLevel(), record.getDataTime(), - record.getDatasetId(), record.getLocation() - .getGridGeometry()); + this(record.getLevel(), record.getDataTime(), record.getDatasetId(), + record.getLocation().getGridGeometry()); } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarDataAccessFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarDataAccessFactory.java new file mode 100644 index 0000000000..1eaf7929c8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarDataAccessFactory.java @@ -0,0 +1,852 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.radar.dataaccess; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.geotools.coverage.grid.GridGeometry2D; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.opengis.referencing.FactoryException; + +import com.raytheon.uf.common.dataaccess.IDataRequest; +import com.raytheon.uf.common.dataaccess.exception.DataRetrievalException; +import com.raytheon.uf.common.dataaccess.exception.EnvelopeProjectionException; +import com.raytheon.uf.common.dataaccess.exception.IncompatibleRequestException; +import com.raytheon.uf.common.dataaccess.exception.InvalidIdentifiersException; +import com.raytheon.uf.common.dataaccess.geom.IGeometryData; +import com.raytheon.uf.common.dataaccess.grid.IGridData; +import com.raytheon.uf.common.dataaccess.impl.AbstractGridDataPluginFactory; +import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData; +import com.raytheon.uf.common.dataaccess.impl.DefaultGridData; +import com.raytheon.uf.common.dataaccess.util.PDOUtil; +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.level.Level; +import com.raytheon.uf.common.dataplugin.level.MasterLevel; +import com.raytheon.uf.common.dataplugin.radar.RadarRecord; +import com.raytheon.uf.common.dataplugin.radar.projection.RadarProjectionFactory; +import com.raytheon.uf.common.dataplugin.radar.util.RadarDataRetriever; +import com.raytheon.uf.common.dataplugin.radar.util.RadarInfo; +import com.raytheon.uf.common.dataplugin.radar.util.RadarInfoDict; +import com.raytheon.uf.common.dataplugin.radar.util.RadarUtil; +import com.raytheon.uf.common.dataquery.requests.DbQueryRequest; +import com.raytheon.uf.common.dataquery.requests.RequestConstraint; +import com.raytheon.uf.common.dataquery.requests.RequestConstraint.ConstraintType; +import com.raytheon.uf.common.dataquery.responses.DbQueryResponse; +import com.raytheon.uf.common.datastorage.DataStoreFactory; +import com.raytheon.uf.common.datastorage.IDataStore; +import com.raytheon.uf.common.datastorage.StorageException; +import com.raytheon.uf.common.geospatial.util.SubGridGeometryCalculator; +import com.raytheon.uf.common.localization.PathManagerFactory; +import com.raytheon.uf.common.numeric.buffer.ByteBufferWrapper; +import com.raytheon.uf.common.numeric.buffer.ShortBufferWrapper; +import com.raytheon.uf.common.numeric.filter.FillValueFilter; +import com.raytheon.uf.common.numeric.source.DataSource; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.Envelope; + +/** + * + * A data factory for getting radar data from the metadata database. There are + * currently not any required identifiers. + * + * This class handles requests for both grid data (i.e., radial and raster + * products) and geometry data (i.e., graphics products). + * + * Radar does not return subgrids for request envelopes like other gridded + * types. Instead data for only icaos within the request envelope are returned + * and all data for the product is used. This is done because subgridding radial + * products is complex and this is not often what a caller actually wants. + * + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date          Ticket#  Engineer    Description
+ * ------------- -------- ----------- --------------------------
+ * Jan 23, 2013           bsteffen    Initial creation
+ * Feb 14, 2013  1614     bsteffen    Refactor data access framework to use
+ *                                    single request.
+ * Feb 04, 2014  2672     bsteffen    Enable requesting icaos within envelope.
+ * Jul 30, 2014  3184     njensen     Overrode optional identifiers
+ * Oct 28, 2014  3755     nabowle     Implement getAvailableParameters, handle
+ *                                    empty parameters, fix error message, and
+ *                                    handle dataless radial radars.
+ * Dec 18, 2014  3600     nabowle     Implement getAvailableLevels and add
+ *                                    optional identifiers to indicate what
+ *                                    fields are used for the level one and two
+ *                                    values.
+ * Feb 13, 2015  4124     mapeters    Inherits IDataFactory.
+ * Feb 27, 2015  4179     mapeters    Use AbstractDataPluginFactory.getAvailableValues().
+ * Apr 18, 2016  5587     tgurney     Implement getIdentifierValues()
+ * Jun 07, 2016  5587     tgurney     Change get*Identifiers() to take
+ *                                    IDataRequest
+ * Jun 08, 2016  5574     mapeters    Add advanced query support, throw exception for
+ *                                    invalid level field identifiers
+ * Aug 01, 2016  2416     tgurney     Add dataURI as optional identifier
+ * Aug 29, 2016  2671     tgurney     Add support for melting layer graphics
+ * Aug 31, 2016  2671     tgurney     Add mesocyclone support
+ * Sep 09, 2016  2671     tgurney     Add storm track (STI) support
+ * Sep 27, 2016  2671     tgurney     Add hail index support
+ * Sep 28, 2016  2671     tgurney     Add tornado vortex sig (TVS) support
+ * Mar 06, 2017  6142     bsteffen    Remove dataURI as optional identifier
+ *
+ *
+ * + * @author bsteffen + */ +public class RadarDataAccessFactory extends AbstractGridDataPluginFactory { + + private static final transient IUFStatusHandler statusHandler = UFStatus + .getHandler(RadarDataAccessFactory.class); + + private static final String PRODUCT_CODE = "productCode"; + + private static final String PRIMARY_ANGLE = "primaryElevationAngle"; + + private static final String TRUE_ANGLE = "trueElevationAngle"; + + private static final String ELEVATION_NUMBER = "elevationNumber"; + + private static final String ICAO = "icao"; + + private static final String LONGITUDE = "longitude"; + + private static final String LATITUDE = "latitude"; + + private static final String FORMAT = "format"; + + private static final String RADIAL_FORMAT = "Radial"; + + private static final String RASTER_FORMAT = "Raster"; + + private static final String GRAPHIC_FORMAT = "Graphic"; + + private static final String LEVEL_ONE = "level.one.field"; + + private static final String LEVEL_TWO = "level.two.field"; + + private static final List SUPPORTED_GRID_FORMATS = Arrays + .asList(RADIAL_FORMAT, RASTER_FORMAT); + + private static final List SUPPORTED_GEOMETRY_FORMATS = Arrays + .asList(GRAPHIC_FORMAT); + + private static final List SUPPORTED_FORMATS = Arrays + .asList(RADIAL_FORMAT, RASTER_FORMAT, GRAPHIC_FORMAT); + + private static final List SUPPORTED_LEVELS = Arrays + .asList(PRIMARY_ANGLE, TRUE_ANGLE, ELEVATION_NUMBER); + + private static final String LEVEL_ERROR = " must be " + PRIMARY_ANGLE + ", " + + TRUE_ANGLE + ", or " + ELEVATION_NUMBER; + + private static RadarInfoDict radarInfo = null; + + private static MasterLevel tiltLevel = null; + + @Override + protected IGridData constructGridDataResponse(IDataRequest request, + PluginDataObject pdo, GridGeometry2D gridGeometry, + DataSource dataSource) { + RadarRecord radarRecord = asRadarRecord(pdo); + RadarInfo radarInfo = getRadarInfo() + .getInfo(radarRecord.getProductCode()); + if (!SUPPORTED_GRID_FORMATS.contains(radarInfo.getFormat())) { + throw new DataRetrievalException(radarInfo.getFormat() + + " data cannot be requested as grid"); + } + DefaultGridData defaultGridData = new DefaultGridData(dataSource, + gridGeometry); + defaultGridData.setDataTime(pdo.getDataTime()); + // reverse map parameter to match request. + List requestedParameters = Arrays + .asList(request.getParameters()); + if (requestedParameters.contains(radarInfo.getName())) { + defaultGridData.setParameter(radarInfo.getName()); + } else if (requestedParameters.contains(radarInfo.getMnemonic())) { + defaultGridData.setParameter(radarRecord.getMnemonic()); + } else { + defaultGridData + .setParameter(radarRecord.getProductCode().toString()); + } + defaultGridData.setUnit(radarRecord.getDataUnit()); + defaultGridData.setLevel(getLevel(radarRecord, request)); + defaultGridData.setLocationName(generateLocationName(radarRecord)); + + Map attributes = new HashMap<>(); + attributes.put(ICAO, radarRecord.getIcao()); + attributes.put(FORMAT, radarRecord.getFormat()); + + defaultGridData.setAttributes(attributes); + + return defaultGridData; + } + + @Override + protected IGeometryData[] getGeometryData(IDataRequest request, + DbQueryResponse dbQueryResponse) { + + Map> results = unpackResults(dbQueryResponse); + List rval = new ArrayList<>(); + + for (Entry> resultEntry : results.entrySet()) { + IDataStore ds = DataStoreFactory.getDataStore(resultEntry.getKey()); + for (RadarRecord radarRecord : resultEntry.getValue()) { + RadarInfo radarInfo = getRadarInfo() + .getInfo(radarRecord.getProductCode()); + + if (!SUPPORTED_GEOMETRY_FORMATS + .contains(radarInfo.getFormat())) { + throw new DataRetrievalException(radarInfo.getFormat() + + " data cannot be requested as geometry"); + } + + try { + RadarDataRetriever.populateRadarRecord(ds, radarRecord); + } catch (FileNotFoundException | StorageException e) { + throw new DataRetrievalException(e.getLocalizedMessage(), + e); + } + + DefaultGeometryData[] geomDatas = null; + // TODO: Update this when more radar graphics are supported + if (radarInfo.getProductCode() == 166) { + geomDatas = RadarGeometryDataUtil + .makeMeltingLayerGeom(radarRecord); + } else if (radarInfo.getProductCode() == 141) { + geomDatas = RadarGeometryDataUtil + .makeMesocycloneGeom(radarRecord); + } else if (radarInfo.getProductCode() == 58) { + geomDatas = RadarGeometryDataUtil + .makeStormTrackGeom(radarRecord); + } else if (radarInfo.getProductCode() == 59) { + geomDatas = RadarGeometryDataUtil + .makeHailIndexGeom(radarRecord); + } else if (radarInfo.getProductCode() == 61) { + geomDatas = RadarGeometryDataUtil.makeTVSGeom(radarRecord); + } else { + throw new DataRetrievalException( + "Product code " + radarInfo.getProductCode() + + " is not yet supported."); + } + // reverse map parameter to match request. + List requestedParameters = Arrays + .asList(request.getParameters()); + String productCode = radarRecord.getProductCode().toString(); + for (DefaultGeometryData geomData : geomDatas) { + if (requestedParameters.contains(radarInfo.getName())) { + geomData.addData(radarInfo.getName(), + radarInfo.getName()); + } else if (requestedParameters + .contains(radarInfo.getMnemonic())) { + geomData.addData(radarInfo.getMnemonic(), + radarInfo.getMnemonic()); + } else { + geomData.addData(productCode, productCode); + } + + // set remaining metadata + geomData.setDataTime(radarRecord.getDataTime()); + geomData.setLevel(getLevel(radarRecord, request)); + geomData.setLocationName(generateLocationName(radarRecord)); + geomData.addAttribute(ICAO, radarRecord.getIcao()); + geomData.addAttribute(FORMAT, radarRecord.getFormat()); + rval.add(geomData); + } + } + } + return rval.toArray(new IGeometryData[rval.size()]); + } + + /** + * Unpack records from response and group by HDF5 file + * + * @param dbQueryResponse + * @return + */ + private static Map> unpackResults( + DbQueryResponse dbQueryResponse) { + // Bin up requests to the same hdf5 + Map> fileMap = new HashMap<>(); + + for (Map result : dbQueryResponse.getResults()) { + Object object = result.get(null); + if (object == null || !(object instanceof RadarRecord)) { + statusHandler.warn( + "Unexpected DB query result for radar: " + object); + continue; + } + RadarRecord record = (RadarRecord) object; + File hdf5File = PDOUtil.getHDF5File(record); + List recList = fileMap.get(hdf5File); + if (recList == null) { + recList = new ArrayList<>(); + fileMap.put(hdf5File, recList); + } + recList.add(record); + } + return fileMap; + } + + /** + * Get the level for the radar record. If the request specifies + * {@value #LEVEL_ONE} or {@value #LEVEL_TWO} identifiers, those fields will + * be used. If {@value #LEVEL_ONE} is not specified, {@value #PRIMARY_ANGLE} + * will be used. + * + * @param radarRecord + * The radar record. + * @param request + * The request. + * @return The created level. + */ + private Level getLevel(RadarRecord radarRecord, IDataRequest request) { + String levelOneField = getLevelField(request, LEVEL_ONE); + String levelTwoField = getLevelField(request, LEVEL_TWO); + + if (levelOneField == null) { + levelOneField = PRIMARY_ANGLE; + } + + Level level; + if (PRIMARY_ANGLE.equals(levelOneField)) { + level = getTiltLevel(radarRecord.getPrimaryElevationAngle()); + } else if (TRUE_ANGLE.equals(levelOneField)) { + level = getTiltLevel(radarRecord.getTrueElevationAngle()); + } else { // elevationNumber + level = new Level(); + level.setMasterLevel(new MasterLevel("OSEQD")); + level.setLevelonevalue(radarRecord.getElevationNumber()); + } + + if (levelTwoField != null) { + if (PRIMARY_ANGLE.equals(levelTwoField)) { + level.setLeveltwovalue(radarRecord.getPrimaryElevationAngle()); + } else if (TRUE_ANGLE.equals(levelTwoField)) { + level.setLeveltwovalue(radarRecord.getTrueElevationAngle()); + } else { // elevationNumber + level.setLeveltwovalue(radarRecord.getElevationNumber()); + } + } + + return level; + } + + /** + * Get the level field corresponding to the given key from the request's + * identifiers. + * + * @param request + * @param levelFieldKey + * key indicating level one or level two + * @return the level field (may be null) + */ + private String getLevelField(IDataRequest request, String levelFieldKey) { + Object levelField = request.getIdentifiers().get(levelFieldKey); + if (levelField == null) { + return null; + } else if (levelField instanceof String + && SUPPORTED_LEVELS.contains(levelField)) { + return (String) levelField; + } + + // Nothing else is valid since the value is a DB field name + throw new IncompatibleRequestException( + "'" + levelField.toString() + "' is not a valid level field"); + } + + /** + * Get a unique name describing the location of the radar data. The name + * always includes icao, elevation angle, num bins and num radials. For + * radial data it also includes the first and last angle of the radial data. + * Theoretically two radial geometries could have the same name but + * internally different angleData but this is very unlikely and the points + * would be very nearly identical. + * + * + * @param radarRecord + * a record. + * @return A unique location name + */ + protected String generateLocationName(RadarRecord radarRecord) { + StringBuilder locationName = new StringBuilder(32); + locationName.append(radarRecord.getIcao()); + locationName.append("_"); + locationName.append(radarRecord.getTrueElevationAngle()); + locationName.append("_"); + locationName.append(radarRecord.getNumBins()); + locationName.append("_"); + locationName.append(radarRecord.getNumRadials()); + float[] angleData = radarRecord.getAngleData(); + if (angleData != null) { + locationName.append("_"); + locationName.append(angleData[0]); + locationName.append("_"); + locationName.append(angleData[angleData.length - 1]); + } + return locationName.toString(); + } + + protected RadarRecord asRadarRecord(PluginDataObject pdo) { + if (pdo instanceof RadarRecord) { + return (RadarRecord) pdo; + } + throw new DataRetrievalException(this.getClass().getSimpleName() + + " cannot handle " + pdo.getClass().getSimpleName()); + } + + @Override + protected GridGeometry2D getGridGeometry(PluginDataObject pdo) { + RadarRecord radarRecord = asRadarRecord(pdo); + if (radarRecord.getFormat().equals(RADIAL_FORMAT)) { + try { + float[] angleData = radarRecord.getAngleData(); + if (angleData == null) { + populateRecord(radarRecord); + angleData = radarRecord.getAngleData(); + } + if (angleData == null) { + return null; + } + + // NOTE: do not set swapXY=true even though it matches the raw + // data better because there is lots of code, especially on the + // Viz side that does not correctly handle the resulting + // GridGeometry. + return RadarProjectionFactory.constructGridGeometry( + new Coordinate(radarRecord.getLongitude(), + radarRecord.getLatitude()), + angleData, radarRecord.getGateResolution(), + radarRecord.getTrueElevationAngle(), + radarRecord.getNumBins(), false); + } catch (FactoryException e) { + throw new DataRetrievalException(e); + } + } else if (radarRecord.getFormat().equals(RASTER_FORMAT)) { + double maxExtent = RadarUtil.calculateExtent(radarRecord); + return RadarUtil.constructGridGeometry(radarRecord.getCRS(), + maxExtent, Math.max(radarRecord.getNumBins(), + radarRecord.getNumRadials())); + + } else { + return super.getGridGeometry(pdo); + } + } + + @Override + protected SubGridGeometryCalculator calculateSubGrid( + ReferencedEnvelope envelope, GridGeometry2D gridGeometry) + throws EnvelopeProjectionException { + /* + * The SubGridGeometryCalculator cannot accurately calculate subgrids + * into RadialBin projections. For this factory the request envelope is + * only used to limit the sites, not to subgrid. Returning null causes + * the super class to request a full grid. + */ + return null; + } + + @Override + protected DataSource getDataSource(PluginDataObject pdo, + SubGridGeometryCalculator subGrid) { + RadarRecord radarRecord = asRadarRecord(pdo); + DataSource dataSource = getDataSource(radarRecord); + if (dataSource == null) { + /* + * Radial data prepopulates the record to get the gridGeometry but + * raster data waits until now. + */ + populateRecord(radarRecord); + dataSource = getDataSource(radarRecord); + if (dataSource == null) { + throw new DataRetrievalException( + "No grid data found for " + radarRecord); + } + } + if (radarRecord.getFormat().equals(RADIAL_FORMAT)) { + /* + * The raw data is in bin,radial format but the grid geometries we + * use are radial,bin so need to do some swapping. + */ + dataSource = new AxisSwapDataSource(dataSource, + radarRecord.getNumBins()); + } + + return dataSource; + } + + /** + * Populate a DataSource from the raw data(byte or short) in the provided + * record. + * + * @param radarRecord + * @return a DataSource or null if the record is not populated or has no + * grid data. + */ + private DataSource getDataSource(RadarRecord radarRecord) { + int nx = radarRecord.getNumBins(); + int ny = radarRecord.getNumRadials(); + byte[] bytes = radarRecord.getRawData(); + if (bytes != null) { + ByteBufferWrapper wrapper = new ByteBufferWrapper(bytes, nx, ny); + return FillValueFilter.apply((DataSource) wrapper, 0); + } + short[] shorts = radarRecord.getRawShortData(); + if (shorts != null) { + ShortBufferWrapper wrapper = new ShortBufferWrapper(shorts, nx, ny); + return FillValueFilter.apply((DataSource) wrapper, 0); + } + return null; + + } + + protected void populateRecord(RadarRecord radarRecord) + throws DataRetrievalException { + try { + RadarDataRetriever.populateRadarRecord( + PDOUtil.getDataStore(radarRecord), radarRecord); + } catch (Exception e) { + throw new DataRetrievalException(e); + } + } + + @Override + protected Map buildConstraintsFromRequest( + IDataRequest request) { + Map constraints = new HashMap<>(); + if (request.getParameters() != null + && request.getParameters().length > 0) { + Set codes = new HashSet<>(); + for (String parameter : request.getParameters()) { + codes.addAll(getProductCodesFromParameter(parameter)); + } + RequestConstraint pcConstraint = new RequestConstraint(null, + ConstraintType.IN); + for (Integer code : codes) { + pcConstraint.addToConstraintValueList(code.toString()); + } + constraints.put(PRODUCT_CODE, pcConstraint); + } + Level[] levels = request.getLevels(); + if (levels != null && levels.length > 0) { + RequestConstraint angleConstraint = new RequestConstraint(null, + ConstraintType.IN); + RequestConstraint levelTwoConstraint = new RequestConstraint(null, + ConstraintType.IN); + + String levelOneField = getLevelField(request, LEVEL_ONE); + String levelTwoField = getLevelField(request, LEVEL_TWO); + + if (levelOneField == null) { + levelOneField = PRIMARY_ANGLE; + } + for (Level level : levels) { + angleConstraint.addToConstraintValueList( + level.getLevelOneValueAsString()); + if (levelTwoField != null + && level.getLeveltwovalue() != Level.INVALID_VALUE) { + levelTwoConstraint.addToConstraintValueList( + level.getLevelTwoValueAsString()); + } + } + constraints.put(levelOneField, angleConstraint); + if (levelTwoConstraint.getConstraintValue() != null) { + constraints.put(levelTwoField, levelTwoConstraint); + } + } + + String[] locations = request.getLocationNames(); + if (locations != null && locations.length > 0) { + RequestConstraint rc = new RequestConstraint(locations); + constraints.put(ICAO, rc); + } + + if (request.getEnvelope() != null) { + Envelope envelope = request.getEnvelope(); + + String minLon = Double.toString(envelope.getMinX()); + String maxLon = Double.toString(envelope.getMaxX()); + constraints.put(LONGITUDE, new RequestConstraint(minLon, maxLon)); + String minLat = Double.toString(envelope.getMinY()); + String maxLat = Double.toString(envelope.getMaxY()); + constraints.put(LATITUDE, new RequestConstraint(minLat, maxLat)); + } + + Map identifiers = request.getIdentifiers(); + if (identifiers != null && identifiers.containsKey(ICAO)) { + Object value = identifiers.get(ICAO); + if (value instanceof RequestConstraint) { + constraints.put(ICAO, (RequestConstraint) value); + } else { + constraints.put(ICAO, new RequestConstraint(value.toString())); + } + } + + return constraints; + } + + private Set getProductCodesFromParameter(String parameter) { + String exception = null; + Set codes = new HashSet<>(); + for (RadarInfo info : getRadarInfo()) { + if (parameter.equals(info.getName()) + || parameter.equals(info.getMnemonic()) || parameter + .equals(Integer.toString(info.getProductCode()))) { + + if (SUPPORTED_FORMATS.contains(info.getFormat())) { + codes.add(info.getProductCode()); + } else { + exception = info.getFormat() + " is not supported"; + } + } + } + if (codes.isEmpty()) { + // If any valid product codes are found then don't complain. + if (exception != null) { + throw new DataRetrievalException(exception); + } else { + throw new DataRetrievalException( + parameter + " is not a valid radar parameter."); + } + } + return codes; + } + + private static synchronized Level getTiltLevel(double angle) { + if (tiltLevel == null) { + tiltLevel = new MasterLevel("TILT"); + tiltLevel.setUnitString("°"); + tiltLevel.setType("INC"); + tiltLevel.setDescription("Tilt angle of a radar scan."); + } + Level level = new Level(); + level.setMasterLevel(tiltLevel); + level.setLevelonevalue(angle); + return level; + } + + private static synchronized RadarInfoDict getRadarInfo() { + if (radarInfo == null) { + File file = PathManagerFactory.getPathManager() + .getStaticFile("radarInfo.txt"); + if (file != null) { + radarInfo = RadarInfoDict.getInstance(file.getParent()); + } + } + return radarInfo; + } + + @Override + public String[] getAvailableLocationNames(IDataRequest request) { + return getAvailableValues(request, ICAO, String.class); + } + + /** + * Get the available parameters for {@link #SUPPORTED_FORMATS supported + * formats}. + */ + @Override + public String[] getAvailableParameters(IDataRequest request) { + DbQueryRequest dbQueryRequest = buildDbQueryRequest(request); + dbQueryRequest.addConstraint(FORMAT, + new RequestConstraint(SUPPORTED_FORMATS)); + dbQueryRequest.setDistinct(Boolean.TRUE); + dbQueryRequest.addRequestField(PRODUCT_CODE); + + DbQueryResponse dbQueryResponse = this + .executeDbQueryRequest(dbQueryRequest, request.toString()); + Set productCodes = new TreeSet<>(); + for (Map result : dbQueryResponse.getResults()) { + productCodes.add((Integer) result.get(PRODUCT_CODE)); + } + Set parameters = new HashSet<>(); + for (RadarInfo info : getRadarInfo()) { + if (productCodes.contains(Integer.valueOf(info.getProductCode()))) { + parameters.add(info.getName()); + parameters.add(info.getMnemonic()); + parameters.add(Integer.toString(info.getProductCode())); + } + } + + return parameters.toArray(new String[0]); + } + + /** + * Get the available levels. The optional identifiers, {@value #LEVEL_ONE} + * and {@value #LEVEL_TWO} can be supplied to choose which level fields are + * returned, otherwise {@value #PRIMARY_ANGLE} will be returned as the level + * one value. + */ + @Override + public Level[] getAvailableLevels(IDataRequest request) { + DbQueryRequest dbQueryRequest = buildDbQueryRequest(request); + dbQueryRequest.setDistinct(Boolean.TRUE); + + String levelOneField = getLevelField(request, LEVEL_ONE); + String levelTwoField = getLevelField(request, LEVEL_TWO); + + if (levelOneField == null) { + levelOneField = PRIMARY_ANGLE; + } + dbQueryRequest.addRequestField(levelOneField); + if (levelTwoField != null) { + dbQueryRequest.addRequestField(levelTwoField); + } + + DbQueryResponse dbQueryResponse = this + .executeDbQueryRequest(dbQueryRequest, request.toString()); + Level level; + List levels = new ArrayList<>(); + for (Map result : dbQueryResponse.getResults()) { + if (PRIMARY_ANGLE.equals(levelOneField) + || TRUE_ANGLE.equals(levelTwoField)) { + level = getTiltLevel( + Double.valueOf(result.get(levelOneField).toString())); + } else { + level = new Level(); + level.setMasterLevel(new MasterLevel("OSEQD")); + level.setLevelonevalue( + Double.valueOf(result.get(levelOneField).toString())); + } + if (levelTwoField != null) { + level.setLeveltwovalue( + Double.valueOf(result.get(levelTwoField).toString())); + } + levels.add(level); + } + + return levels.toArray(new Level[0]); + } + + @Override + public String[] getOptionalIdentifiers(IDataRequest request) { + return new String[] { ICAO, LEVEL_ONE, LEVEL_TWO }; + } + + @Override + protected DbQueryRequest buildDbQueryRequest(IDataRequest request) { + validateLevelIdentifiers(request); + return super.buildDbQueryRequest(request); + } + + /** + * Validates that, if specified, the {@value #LEVEL_ONE} and + * {@value #LEVEL_TWO} identifier values are supported. + * + * @param request + */ + private void validateLevelIdentifiers(IDataRequest request) { + String levelOneField = getLevelField(request, LEVEL_ONE); + String levelTwoField = getLevelField(request, LEVEL_TWO); + + if (levelOneField != null + && !SUPPORTED_LEVELS.contains(levelOneField)) { + throw new DataRetrievalException(LEVEL_ONE + LEVEL_ERROR); + } + + if (levelTwoField != null + && !SUPPORTED_LEVELS.contains(levelTwoField)) { + throw new DataRetrievalException(LEVEL_TWO + LEVEL_ERROR); + } + } + + /** + * Get allowed values for a specified identifier + * + * @param request + * Request including the name of the datatype + * @param identifierKey + * The identifier to get allowed values for + */ + @Override + public String[] getIdentifierValues(IDataRequest request, + String identifierKey) { + if (!Arrays.asList(getRequiredIdentifiers(request)) + .contains(identifierKey) + && !Arrays.asList(getOptionalIdentifiers(request)) + .contains(identifierKey)) { + throw new InvalidIdentifiersException(request.getDatatype(), null, + Arrays.asList(new String[] { identifierKey })); + } + List idValStrings; + if (identifierKey.equals(LEVEL_ONE) + || identifierKey.equals(LEVEL_TWO)) { + idValStrings = SUPPORTED_LEVELS; + } else { + Object[] idValues = getAvailableValues(request, identifierKey, + Object.class); + idValStrings = new ArrayList<>(idValues.length); + for (Object idValue : idValues) { + idValStrings.add(idValue.toString()); + } + } + return idValStrings.toArray(new String[idValStrings.size()]); + } + + /** + * + * This is used to convert data from bin,radial format to radial bin format. + * + * 
+     *
+     * SOFTWARE HISTORY
+     *
+     * Date         Ticket#    Engineer    Description
+     * ------------ ---------- ----------- --------------------------
+     * Jan 25, 2013            bsteffen     Initial creation
+     * Feb 14, 2013 1614       bsteffen    refactor data access framework to use
+     *                                          single request.
+     *
+     *
+ * + * @author bsteffen + * @version 1.0 + */ + private static class AxisSwapDataSource implements DataSource { + + private final DataSource realData; + + private final int numBins; + + public AxisSwapDataSource(DataSource realData, int numBins) { + this.realData = realData; + this.numBins = numBins; + } + + @Override + public double getDataValue(int x, int y) { + return realData.getDataValue(numBins - 1 - y, x); + } + + } +} \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarGeometryDataUtil.java b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarGeometryDataUtil.java new file mode 100644 index 0000000000..4fdea973ef --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/dataaccess/RadarGeometryDataUtil.java @@ -0,0 +1,563 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.radar.dataaccess; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.geotools.coverage.grid.GeneralGridGeometry; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; + +import com.raytheon.uf.common.dataaccess.impl.DefaultGeometryData; +import com.raytheon.uf.common.dataplugin.radar.RadarDataKey; +import com.raytheon.uf.common.dataplugin.radar.RadarDataPoint; +import com.raytheon.uf.common.dataplugin.radar.RadarRecord; +import com.raytheon.uf.common.dataplugin.radar.level3.HdaHailPacket.HdaHailPoint; +import com.raytheon.uf.common.dataplugin.radar.level3.SCITDataPacket; +import com.raytheon.uf.common.dataplugin.radar.level3.SCITDataPacket.SCITDataCell; +import com.raytheon.uf.common.dataplugin.radar.level3.SpecialGraphicSymbolPacket.SpecialGraphicPoint; +import com.raytheon.uf.common.dataplugin.radar.level3.StormIDPacket; +import com.raytheon.uf.common.dataplugin.radar.level3.StormIDPacket.StormIDPoint; +import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket; +import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPoint; +import com.raytheon.uf.common.dataplugin.radar.level3.TVSPacket.TVSPoint; +import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket; +import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.MapValues; +import com.raytheon.uf.common.dataplugin.radar.util.RadarRecordUtil; +import com.raytheon.uf.common.geospatial.ReferencedCoordinate; +import com.raytheon.uf.common.geospatial.ReferencedObject.Type; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.common.util.Pair; +import com.vividsolutions.jts.geom.Coordinate; +import com.vividsolutions.jts.geom.Geometry; +import com.vividsolutions.jts.geom.GeometryCollection; +import com.vividsolutions.jts.geom.GeometryFactory; +import com.vividsolutions.jts.geom.LinearRing; +import com.vividsolutions.jts.geom.MultiPoint; +import com.vividsolutions.jts.geom.Point; + +/** + * Methods that create IGeometryData for radar products + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Aug 30, 2016 2671       tgurney     Initial creation
+ * Aug 31, 2016 2671       tgurney     Add makeMesocycloneGeom
+ * Sep 08, 2016 2671       tgurney     Add makeStormTrackGeom
+ * Sep 27, 2016 2671       tgurney     Add makeHailIndexGeom
+ * Sep 28, 2016 2671       tgurney     Add makeTVSGeom
+ *
+ *
+ * + * @author tgurney + */ + +public class RadarGeometryDataUtil { + + /** + * Data representing storm track for a single storm. + */ + private static class StormTrackData { + /** + * Index of the point corresponding to the storm's current location. All + * points before that are past, and all points after that are forecast. + */ + public Integer currentLocIndex; + + public MultiPoint points; + + public String stormId = ""; + } + + private static final GeometryFactory geomFactory = new GeometryFactory(); + + private static final transient IUFStatusHandler statusHandler = UFStatus + .getHandler(RadarGeometryDataUtil.class); + + private RadarGeometryDataUtil() { + // static interface only + } + + public static DefaultGeometryData[] makeMeltingLayerGeom( + RadarRecord radarRecord) { + Map coords = RadarRecordUtil + .buildMeltingLayerCoordinates(radarRecord); + List rings = new ArrayList<>(); + for (Integer num : coords.keySet()) { + rings.add(geomFactory.createLinearRing(coords.get(num))); + } + GeometryCollection geomCollection = geomFactory + .createGeometryCollection(rings.toArray(new Geometry[0])); + DefaultGeometryData rval = new DefaultGeometryData(); + rval.setGeometry(geomCollection); + return new DefaultGeometryData[] { rval }; + } + + /** + * @return (circleId, point) for specific location, or null if the data for + * that location is nonexistent or invalid. + */ + private static Pair getMesocyclonePoint( + RadarDataKey currLoc, RadarRecord radarRecord) { + Integer productCode = radarRecord.getProductCode(); + RadarDataPoint currStorm = radarRecord.getSymbologyData().get(currLoc); + + HashMap displayPointData = currStorm + .getDisplayPointData().get(productCode); + if (displayPointData == null) { + statusHandler.warn("Mesocyclone at " + currLoc + " has no " + + "points. Expected 1"); + return null; + } + + Collection textPackets = currStorm + .getDisplayPacketData().get(productCode).values(); + if (textPackets.size() != 1) { + statusHandler.warn("Mesocyclone at " + currLoc + " has " + + textPackets.size() + " text packets. Expected 1"); + return null; + } + + SymbologyPacket textPacket = textPackets.iterator().next(); + if (!(textPacket instanceof TextSymbolPacket)) { + statusHandler.warn("Mesocyclone at " + currLoc + ": Expected " + + "a " + TextSymbolPacket.class.getSimpleName() + ", got a " + + textPacket.getClass().getName()); + return null; + } + + Collection currPoints = displayPointData.values(); + if (currPoints.size() != 1) { + statusHandler.warn("Mesocyclone at " + currLoc + " has " + + currPoints.size() + " points. Expected 1"); + return null; + } + + SymbologyPoint currPoint = currPoints.iterator().next(); + if (!(currPoint instanceof SpecialGraphicPoint)) { + statusHandler.warn("Mesocyclone at " + currLoc + ": Expected " + + "a " + SpecialGraphicPoint.class.getSimpleName() + + ", got a " + currPoint.getClass().getName()); + return null; + } + + String circleId = ((TextSymbolPacket) textPacket).getTheText().trim(); + SpecialGraphicPoint point = (SpecialGraphicPoint) currPoint; + return new Pair<>(circleId, point); + } + + /** + * @return One geometry data per mesocyclone. The geometry itself is a + * single point. Radius is stored in the "radiusKm" attribute. Other + * tabular data are stored in the 'tableData' attribute. + */ + public static DefaultGeometryData[] makeMesocycloneGeom( + RadarRecord radarRecord) { + List geomDatas = new ArrayList<>(); + + for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) { + DefaultGeometryData geomData = new DefaultGeometryData(); + Pair mesoPoint = getMesocyclonePoint( + currLoc, radarRecord); + if (mesoPoint == null) { + // failed validation + continue; + } + String circleId = mesoPoint.getFirst(); + SpecialGraphicPoint graphicPoint = mesoPoint.getSecond(); + + // Make coordinate from graphic point + GeneralGridGeometry gg = RadarRecordUtil + .getRadarGraphicsGridGeometry(radarRecord); + Coordinate coord = null; + try { + coord = new ReferencedCoordinate( + RadarRecordUtil.rectifyCoordinate( + new Coordinate(graphicPoint.i, graphicPoint.j)), + gg, Type.GRID_CENTER).asLatLon(); + } catch (TransformException | FactoryException e) { + statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), + e); + continue; + } + + // Get tabular data + Map dataMap = radarRecord.getMapProductVals() + .get(MapValues.MESO_TYPE).get(circleId); + if (dataMap == null) { + statusHandler.warn("Mesocyclone at " + currLoc + + ": No tabular data found"); + continue; + } + Map newDataMap = new HashMap<>(); + for (Entry mesoData : dataMap.entrySet()) { + // Remove MESO_ prefix + newDataMap.put(mesoData.getKey().toString().substring(5), + mesoData.getValue()); + } + // Radius is in 1/4 km; convert to km + double radius = graphicPoint.getPointFeatureAttr() / 4.0; + newDataMap.put("RADIUS_KM", Double.toString(radius)); + geomData.addAttribute("tableData", newDataMap); + geomData.setGeometry(geomFactory.createPoint(coord)); + geomDatas.add(geomData); + } + return geomDatas.toArray(new DefaultGeometryData[0]); + } + + /** + * @return One geometry data per storm. The geometry itself is a multipoint. + * Tabular data is stored in 'tableData' attribute. The current + * location of the storm is the point with index specified by + * "currentLocIndex" attribute. All points before that index are + * past and all points after it are forecast. + */ + public static DefaultGeometryData[] makeStormTrackGeom( + RadarRecord radarRecord) { + List geomDatas = new ArrayList<>(); + + for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) { + + DefaultGeometryData geomData = new DefaultGeometryData(); + // Get storm ID and all points + RadarDataPoint currStorm = radarRecord.getSymbologyData() + .get(currLoc); + StormTrackData stormTrackData = getStormTrackPoints(currStorm, + RadarRecordUtil.getRadarGraphicsGridGeometry(radarRecord)); + String stormId = stormTrackData.stormId; + if ("".equals(stormId)) { + statusHandler + .warn("Storm track at " + currLoc + " has no storm ID"); + continue; + } + + // Get tabular data + Map dataMap = radarRecord.getMapProductVals() + .get(MapValues.STI_TYPE).get(stormId); + if (dataMap == null) { + statusHandler.warn("Storm track " + stormId + " at " + currLoc + + ": No tabular data found"); + continue; + } + Map newDataMap = new HashMap<>(); + for (Entry stiData : dataMap.entrySet()) { + // Remove STI_ prefix + newDataMap.put(stiData.getKey().toString().substring(4), + stiData.getValue()); + } + geomData.addAttribute("currentLocIndex", + stormTrackData.currentLocIndex.toString()); + newDataMap.put("STORM_ID", stormId); + geomData.addAttribute("tableData", newDataMap); + geomData.setGeometry(stormTrackData.points); + geomDatas.add(geomData); + } + + return geomDatas.toArray(new DefaultGeometryData[0]); + } + + public static DefaultGeometryData[] makeHailIndexGeom( + RadarRecord radarRecord) { + List geomDatas = new ArrayList<>(); + + for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) { + + HashMap> displayPointData = radarRecord + .getSymbologyData().get(currLoc).getDisplayPointData(); + + HashMap pointMap = displayPointData + .get(radarRecord.getProductCode()); + if (pointMap == null) { + statusHandler.warn( + "Hail index at " + currLoc + " has no " + "points."); + continue; + } + + // Get coordinate and storm id + GeneralGridGeometry gg = RadarRecordUtil + .getRadarGraphicsGridGeometry(radarRecord); + DefaultGeometryData geomData = new DefaultGeometryData(); + String stormId = ""; + Coordinate coord = null; + for (SymbologyPoint point : pointMap.values()) { + if (point instanceof HdaHailPoint) { + try { + coord = new ReferencedCoordinate( + RadarRecordUtil.rectifyCoordinate( + new Coordinate(((HdaHailPoint) point).i, + ((HdaHailPoint) point).j)), + gg, Type.GRID_CENTER).asLatLon(); + } catch (TransformException | FactoryException e) { + statusHandler.handle(Priority.PROBLEM, + e.getLocalizedMessage(), e); + break; + } + } else if (point instanceof StormIDPoint) { + stormId = ((StormIDPoint) point).getStormID(); + } + } + if (coord == null) { + continue; + } + + // Get tabular data + Map dataMap = radarRecord.getMapProductVals() + .get(MapValues.HAIL_TYPE).get(stormId); + if (dataMap == null) { + statusHandler.warn("Hail index " + stormId + " at " + currLoc + + ": No tabular data found"); + continue; + } + Map newDataMap = new HashMap<>(); + for (Entry stiData : dataMap.entrySet()) { + // Remove HI_ prefix + newDataMap.put(stiData.getKey().toString().substring(3), + stiData.getValue()); + } + newDataMap.put("STORM_ID", stormId); + geomData.addAttribute("tableData", newDataMap); + geomData.setGeometry(geomFactory.createPoint(coord)); + geomDatas.add(geomData); + } + + return geomDatas.toArray(new DefaultGeometryData[0]); + } + + /** + * @return (stormId, point) for specific location, or null if the data for + * that location is nonexistent or invalid. + */ + private static Pair getTVSPoint(RadarDataKey currLoc, + RadarRecord radarRecord) { + Integer productCode = radarRecord.getProductCode(); + RadarDataPoint currStorm = radarRecord.getSymbologyData().get(currLoc); + + HashMap displayPointData = currStorm + .getDisplayPointData().get(productCode); + if (displayPointData == null) { + statusHandler.warn( + "TVS at " + currLoc + " has no " + "points. Expected 1"); + return null; + } + + Collection displayPackets = currStorm + .getDisplayPacketData().get(productCode).values(); + if (displayPackets.size() != 1) { + statusHandler.warn("TVS at " + currLoc + " has " + + displayPackets.size() + " text packets. Expected 1"); + return null; + } + + SymbologyPacket stormIdPacket = displayPackets.iterator().next(); + if (!(stormIdPacket instanceof StormIDPacket)) { + statusHandler.warn("TVS at " + currLoc + ": Expected " + "a " + + StormIDPacket.class.getSimpleName() + ", got a " + + stormIdPacket.getClass().getName()); + return null; + } + + Collection currPoints = displayPointData.values(); + if (currPoints.size() != 1) { + statusHandler.warn("TVS at " + currLoc + " has " + currPoints.size() + + " points. Expected 1"); + return null; + } + + SymbologyPoint currPoint = currPoints.iterator().next(); + if (!(currPoint instanceof TVSPoint)) { + statusHandler.warn("TVS at " + currLoc + ": Expected " + "a " + + TVSPoint.class.getSimpleName() + ", got a " + + currPoint.getClass().getName()); + return null; + } + + StormIDPoint[] stormIdPoints = ((StormIDPacket) stormIdPacket) + .getPoints(); + if (stormIdPoints.length != 1) { + statusHandler.warn("TVS at " + currLoc + " has " + + stormIdPoints.length + " storm ID points. Expected 1"); + return null; + } + TVSPoint point = (TVSPoint) currPoint; + String stormId = stormIdPoints[0].getStormID(); + return new Pair<>(stormId, point); + } + + public static DefaultGeometryData[] makeTVSGeom(RadarRecord radarRecord) { + List geomDatas = new ArrayList<>(); + GeneralGridGeometry gg = RadarRecordUtil + .getRadarGraphicsGridGeometry(radarRecord); + + for (RadarDataKey currLoc : radarRecord.getSymbologyData().keySet()) { + + DefaultGeometryData geomData = new DefaultGeometryData(); + // Get point and storm ID + Pair tvsData = getTVSPoint(currLoc, radarRecord); + if (tvsData == null) { + // failed validation + continue; + } + String stormId = tvsData.getFirst(); + TVSPoint point = tvsData.getSecond(); + Coordinate coord; + try { + coord = new ReferencedCoordinate( + RadarRecordUtil.rectifyCoordinate( + new Coordinate(point.i, point.j)), + gg, Type.GRID_CENTER).asLatLon(); + } catch (TransformException | FactoryException e) { + statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), + e); + continue; + } + + // Get tabular data + Map dataMap = radarRecord.getMapProductVals() + .get(MapValues.TVS_TYPE).values().iterator().next(); + if (dataMap == null) { + statusHandler.warn("TVS " + stormId + " at " + currLoc + + ": No tabular data found"); + continue; + } + Map newDataMap = new HashMap<>(); + for (Entry stiData : dataMap.entrySet()) { + // Remove TVS_ prefix + newDataMap.put(stiData.getKey().toString().substring(4), + stiData.getValue()); + } + newDataMap.put("STORM_ID", stormId); + newDataMap.put("ELEVATED", point.elevated ? "Y" : "N"); + + geomData.addAttribute("tableData", newDataMap); + geomData.setGeometry(geomFactory.createPoint(coord)); + geomDatas.add(geomData); + } + return geomDatas.toArray(new DefaultGeometryData[0]); + } + + private static StormTrackData getStormTrackPoints(RadarDataPoint stiPoint, + GeneralGridGeometry gridGeometry) { + List points = new ArrayList<>(); + SCITDataPacket forecastPacket = null; + SCITDataPacket pastPacket = null; + TextSymbolPacket currentPacket = null; + StormIDPoint stormId = null; + StormTrackData stormTrackData = new StormTrackData(); + + HashMap> displayPacketData = stiPoint + .getDisplayPacketData(); + + for (Integer type : displayPacketData.keySet()) { + for (SymbologyPacket packet : displayPacketData.get(type) + .values()) { + if (packet instanceof SCITDataPacket) { + for (SCITDataCell currCell : ((SCITDataPacket) packet) + .getPoints()) { + if (currCell.getText().contains("!")) { + // '!' indicates past + pastPacket = (SCITDataPacket) packet; + continue; + } else if (currCell.getText().contains("#")) { + // '#' indicates forecast + forecastPacket = (SCITDataPacket) packet; + continue; + } + } + } else if (packet instanceof TextSymbolPacket) { + if (((TextSymbolPacket) packet).getTheText() + .contains("\"")) { + // '"' indicates current storm location + currentPacket = (TextSymbolPacket) packet; + } + } else if (packet instanceof StormIDPacket) { + // Should only have one point + stormId = ((StormIDPacket) packet).getPoints()[0]; + } + } + } + + if (pastPacket != null) { + points.addAll(createSCITDataCell(pastPacket, gridGeometry)); + } + if (currentPacket != null) { + points.addAll(createSCITDataCell(currentPacket, gridGeometry)); + stormTrackData.currentLocIndex = points.size() - 1; + } + if (forecastPacket != null) { + points.addAll(createSCITDataCell(forecastPacket, gridGeometry)); + } + stormTrackData.points = geomFactory + .createMultiPoint(points.toArray(new Point[0])); + if (stormId != null) { + stormTrackData.stormId = stormId.getStormID(); + } + return stormTrackData; + } + + /** + * @return List of points from a single SymbologyPacket + */ + private static List createSCITDataCell(SymbologyPacket packet, + GeneralGridGeometry gridGeometry) { + List points = new ArrayList<>(); + List cells; + if (packet instanceof TextSymbolPacket) { + TextSymbolPacket pkt = (TextSymbolPacket) packet; + cells = new ArrayList<>(); + cells.add(new SCITDataCell()); + cells.get(0).setText(pkt.getTheText()); + cells.get(0).setI(pkt.getI()); + cells.get(0).setJ(pkt.getJ()); + } else { + SCITDataPacket pkt = (SCITDataPacket) packet; + cells = pkt.getPoints(); + } + if (cells == null) { + return points; + } + + for (SCITDataCell point : cells) { + Coordinate coord = null; + try { + coord = new ReferencedCoordinate( + RadarRecordUtil.rectifyCoordinate( + new Coordinate(point.i, point.j)), + gridGeometry, Type.GRID_CENTER).asLatLon(); + } catch (TransformException | FactoryException e) { + statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage(), + e); + continue; + } + points.add(geomFactory.createPoint(coord)); + } + return points; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/level3/SCLPacket.java b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/level3/SCLPacket.java new file mode 100644 index 0000000000..65a1f505b5 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/level3/SCLPacket.java @@ -0,0 +1,55 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.radar.level3; + +import java.io.DataInputStream; +import java.io.IOException; + +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; + +/** + * This is a SCL centric version of the Generic Packet + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 18, 2016 DR 18796   jdynina    Initial creation
+ * Mar 06, 2016 DR 19848   jdynina    Renamed SCC to SCL
+ * 
+ *
+ * + * @author jdynina + * @version 1.0 + */ + +@DynamicSerialize +public class SCLPacket extends GenericDataPacket { + + public SCLPacket(int packetId, DataInputStream in) throws IOException { + super(packetId, in); + } + + public SCLPacket() { + + } + +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java index a7e4a3138c..c66203e16e 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java @@ -1,19 +1,19 @@ /** * This software was developed and / or modified by Raytheon Company, * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * + * * U.S. EXPORT CONTROLLED TECHNICAL DATA * This software product contains export-restricted data whose * export/transfer/disclosure is restricted by U.S. law. Dissemination * to non-U.S. persons whether in the United States or abroad requires * an export license or other authorization. - * + * * Contractor Name: Raytheon Company * Contractor Address: 6825 Pine Street, Suite 340 * Mail Stop B8 * Omaha, NE 68106 * 402.291.0100 - * + * * See the AWIPS II Master Rights File ("Master Rights File.pdf") for * further licensing information. **/ @@ -27,7 +27,12 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; +import org.geotools.coverage.grid.GeneralGridEnvelope; +import org.geotools.coverage.grid.GeneralGridGeometry; +import org.geotools.geometry.GeneralEnvelope; import org.geotools.referencing.GeodeticCalculator; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.operation.TransformException; import com.raytheon.uf.common.dataplugin.radar.RadarDataKey; import com.raytheon.uf.common.dataplugin.radar.RadarDataPoint; @@ -36,6 +41,8 @@ import com.raytheon.uf.common.dataplugin.radar.level3.DMDPacket; import com.raytheon.uf.common.dataplugin.radar.level3.DMDPacket.DMDAttributeIDs; import com.raytheon.uf.common.dataplugin.radar.level3.GraphicBlock; import com.raytheon.uf.common.dataplugin.radar.level3.Layer; +import com.raytheon.uf.common.dataplugin.radar.level3.LinkedContourVectorPacket; +import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVector; import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyBlock; import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket; import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket; @@ -44,61 +51,74 @@ import com.raytheon.uf.common.dataplugin.radar.level3.generic.AreaComponent.Area import com.raytheon.uf.common.dataplugin.radar.level3.generic.GenericDataComponent; import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.DHRValues; import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.GraphicBlockValues; +import com.raytheon.uf.common.geospatial.ReferencedCoordinate; +import com.raytheon.uf.common.geospatial.ReferencedObject.Type; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; import com.vividsolutions.jts.geom.Coordinate; /** * Functions on radar record for retrieval and manipulation of data - * + * * 
- * 
+ *
  * SOFTWARE HISTORY
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
- * Aug 11, 2010            mnash     Initial creation
- * Dec 28, 2011 11705	   gzhang	 Fix SCAN missing Rows error	
+ * Aug 11, 2010            mnash       Initial creation
+ * Dec 28, 2011 11705      gzhang      Fix SCAN missing Rows error
  * Mar 19, 2013 1804       bsteffen    Reduce useless data stored in radar hdf5
  * Mar 19, 2013 1804       bsteffen    Remove empty data structures from radar
  *                                     hdf5.
- * Sep 03, 2013 DR 13083   gzhang    Add DHR Bias support for ADAD(38)/(46).
+ * Sep 03, 2013 DR 13083   gzhang      Add DHR Bias support for ADAD(38)/(46).
+ * Aug 26, 2016 2671       tgurney     Extract buildMeltingLayerCoordinates from
+ *                                     RadarMLResource
+ * Aug 31, 2016 2671       tgurney     Make rectifyCoordinate public. Extract
+ *                                     new method getRadarGridGeometry from
+ *                                     buildMeltingLayerCoordinates
  *
- * + * * @author mnash - * @version 1.0 */ public class RadarRecordUtil { private static final String NO_STORMS_DETECTED = "NO STORMS DETECTED"; + private static final transient IUFStatusHandler statusHandler = UFStatus + .getHandler(RadarRecordUtil.class); + public static Map> parseGraphicBlock( RadarRecord record) { - Map> values = new LinkedHashMap>(); + Map> values = new LinkedHashMap<>(); if (record != null) { GraphicBlock gb = record.getGraphicBlock(); if (gb != null) { Layer[] pages = gb.getPages(); - for (int i = 0; i < pages.length; i++) { - SymbologyPacket[] packets = pages[i].getPackets(); + for (Layer page : pages) { + SymbologyPacket[] packets = page.getPackets(); for (int j = 1; j < packets.length; j++) { if (packets[j] instanceof TextSymbolPacket) { if (!NO_STORMS_DETECTED .equals(((TextSymbolPacket) packets[j]) .getTheText())) { - Map map = new HashMap(); + Map map = new HashMap<>(); Matcher m = RadarConstants.graphic_block_pattern - .matcher(getNormalizedGBText(((TextSymbolPacket) packets[j]) - .getTheText())); + .matcher(getNormalizedGBText( + ((TextSymbolPacket) packets[j]) + .getTheText())); if (m.find()) { String storm_id = m.group(1).trim(); - map.put(GraphicBlockValues.AZIMUTH, m - .group(2).trim()); - map.put(GraphicBlockValues.RANGE, m - .group(3).trim()); - map.put(GraphicBlockValues.TVS, m.group(4) - .trim()); - map.put(GraphicBlockValues.MDA, m.group(5) - .trim()); + map.put(GraphicBlockValues.AZIMUTH, + m.group(2).trim()); + map.put(GraphicBlockValues.RANGE, + m.group(3).trim()); + map.put(GraphicBlockValues.TVS, + m.group(4).trim()); + map.put(GraphicBlockValues.MDA, + m.group(5).trim()); String temp = m.group(6).trim(); if ("UNKNOWN".equals(temp)) { map.put(GraphicBlockValues.POSH, temp); @@ -113,14 +133,14 @@ public class RadarRecordUtil { map.put(GraphicBlockValues.MXHAILSIZE, temp.split("/")[2].trim()); } - map.put(GraphicBlockValues.VIL, m.group(7) - .trim()); - map.put(GraphicBlockValues.DBZM, m.group(8) - .trim()); - map.put(GraphicBlockValues.HT, m.group(9) - .trim()); - map.put(GraphicBlockValues.TOP, m.group(10) - .trim()); + map.put(GraphicBlockValues.VIL, + m.group(7).trim()); + map.put(GraphicBlockValues.DBZM, + m.group(8).trim()); + map.put(GraphicBlockValues.HT, + m.group(9).trim()); + map.put(GraphicBlockValues.TOP, + m.group(10).trim()); temp = m.group(11).trim(); if ("NEW".equals(temp)) { map.put(GraphicBlockValues.FCSTDIR, @@ -152,7 +172,8 @@ public class RadarRecordUtil { for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) { RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon); - for (Integer type : currPoint.getDisplayGenericPointData().keySet()) { + for (Integer type : currPoint.getDisplayGenericPointData() + .keySet()) { for (GenericDataComponent currComponent : currPoint .getDisplayGenericPointData().get(type).values()) { if (featureId.equalsIgnoreCase(currComponent @@ -168,12 +189,12 @@ public class RadarRecordUtil { /** * Get the GenericDataComponent. - * + * * @param record * The RadarRecord * @param featureId * The featureId - * + * * @return The GenericDataComponent, or null if no matches */ public static GenericDataComponent getFeatureValues(RadarRecord record, @@ -181,7 +202,8 @@ public class RadarRecordUtil { for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) { RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon); - for (Integer type : currPoint.getDisplayGenericPointData().keySet()) { + for (Integer type : currPoint.getDisplayGenericPointData() + .keySet()) { for (GenericDataComponent currComponent : currPoint .getDisplayGenericPointData().get(type).values()) { if (featureId.equalsIgnoreCase(currComponent @@ -195,12 +217,13 @@ public class RadarRecordUtil { } public static List getDMDFeatureIDs(RadarRecord record) { - List rval = new ArrayList(); + List rval = new ArrayList<>(); SymbologyBlock sb = record.getSymbologyBlock(); if (sb != null) { for (Layer layer : sb.getLayers()) { - for (SymbologyPacket packet : layer.getPackets()) + for (SymbologyPacket packet : layer.getPackets()) { rval.addAll(((DMDPacket) packet).getFeatureIDs()); + } } } return rval; @@ -213,7 +236,8 @@ public class RadarRecordUtil { for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) { RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon); - for (Integer type : currPoint.getDisplayGenericPointData().keySet()) { + for (Integer type : currPoint.getDisplayGenericPointData() + .keySet()) { for (GenericDataComponent currComponent : currPoint .getDisplayGenericPointData().get(type).values()) { currFeature = (AreaComponent) currComponent; @@ -233,18 +257,18 @@ public class RadarRecordUtil { /** * Search in the map for the storm id and send back the feature if that is * the case - * + * * @param type * @param id * @return */ public List getFeatures(RadarRecord record, RadarConstants.MapValues type, String id) { - List list = new ArrayList(); + List list = new ArrayList<>(); for (Map.Entry> entry : record .getMapProductVals().get(type).entrySet()) { - String fid = entry.getValue().get( - RadarConstants.MapValues.MESO_STORM_ID); + String fid = entry.getValue() + .get(RadarConstants.MapValues.MESO_STORM_ID); if (fid != null) { fid = fid.trim(); if (id.trim().equals(fid)) { @@ -285,18 +309,19 @@ public class RadarRecordUtil { GeodeticCalculator gd = new GeodeticCalculator(); gd.setStartingGeographicPoint(lng, lat); gd.setDirection(calcDir, rng); - Coordinate coor = new Coordinate(gd.getDestinationGeographicPoint() - .getX(), gd.getDestinationGeographicPoint().getY()); + Coordinate coor = new Coordinate( + gd.getDestinationGeographicPoint().getX(), + gd.getDestinationGeographicPoint().getY()); return coor; } /** - * + * * Calculates the 8-bit SRM product from 8-bit base velocity. SRM is found * by subtracting the overall wind direction from the radial data. The * resulting data would show the velocity data as if the wind was * stationary. - * + * * @param record * @param direction * @param speed @@ -317,14 +342,15 @@ public class RadarRecordUtil { int maxBin = 0; // Loop through each bin, in each radial - for (int currRadial = 0; currRadial < record.getNumRadials(); currRadial++) { + for (int currRadial = 0; currRadial < record + .getNumRadials(); currRadial++) { // If it is moving, find the Integer delta value if (record.srmSpeed != 0) { // Get the delta value for the current radial delta = record.srmSpeed - * Math.cos((Math.PI / 180) - * (record.srmDirection - record - .getAngleData()[currRadial])) + * Math.cos(Math.PI / 180 + * (record.srmDirection + - record.getAngleData()[currRadial])) / 1.944; if (!record.isExpandedMode()) { @@ -356,7 +382,8 @@ public class RadarRecordUtil { record.srmData[currBin] = (byte) 1; } else if (radialData[currBin] != 0) { // Add delta to the radialPixel - record.srmData[currBin] = (byte) (radialData[currBin] + deltaInt); + record.srmData[currBin] = (byte) (radialData[currBin] + + deltaInt); } } } @@ -408,13 +435,14 @@ public class RadarRecordUtil { RadarRecordUtil.calculateSRM8(record); } - public static byte getSRMDataValue(RadarRecord record, int radial, int bin) { + public static byte getSRMDataValue(RadarRecord record, int radial, + int bin) { if (record.srmData == null) { RadarRecordUtil.calculateSRM8(record); } - return (record.srmData != null) ? record.srmData[radial - * record.getNumBins() + bin] : 0; + return record.srmData != null + ? record.srmData[radial * record.getNumBins() + bin] : 0; } public static boolean hasSRM(RadarRecord record) { @@ -440,7 +468,7 @@ public class RadarRecordUtil { DHRValues.BIASAPPLIEDFLAG }; public static Map getDHRValues(RadarRecord record) { - Map map = new HashMap(); + Map map = new HashMap<>(); String text = null; SymbologyBlock sb = record.getSymbologyBlock(); if (sb != null) { @@ -469,8 +497,9 @@ public class RadarRecordUtil { Integer flagZeroHybrid = null; String[] v = new String[nv]; - for (vi = 0; vi < nv; ++vi) + for (vi = 0; vi < nv; ++vi) { v[vi] = text.substring(vi * 8, (vi + 1) * 8); + } vi = 0; while (vi < nv) { @@ -480,8 +509,9 @@ public class RadarRecordUtil { map.put(DHRValues.PRECIPCAT, (double) precipCat); vi += 6; } else if (s.equals("ADAP(32)")) { - for (int i = 0; i < ADAP32_VALUES.length; ++i) - map.put(ADAP32_VALUES[i], parseDHRValue(v[vi++])); + for (DHRValues element : ADAP32_VALUES) { + map.put(element, parseDHRValue(v[vi++])); + } biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0; while (vi < nv) { s = v[vi++]; @@ -493,8 +523,9 @@ public class RadarRecordUtil { * parseDHRValue(text, vi + 1)); */ flagZeroHybrid = (int) parseDHRValue(v[vi + 2]); - if (flagZeroHybrid != 0 && flagZeroHybrid != 1) + if (flagZeroHybrid != 0 && flagZeroHybrid != 1) { flagZeroHybrid = 0; // should print warning + } vi += 15; } else if (s.equals("BIAS(11)")) { biasCalculated = parseDHRValue(v[vi + 8]); @@ -506,14 +537,17 @@ public class RadarRecordUtil { // from A1 decodeDHR.C. map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9])); map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10])); - map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25])); - map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[vi + 37])); biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083 + map.put(DHRValues.MAXPRECIPRATEALLOW, + parseDHRValue(v[vi + 25])); + map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[vi + 37])); + biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083 s = v[46]; if (s.equals("SUPL(15)")) { biasCalculated = parseDHRValue(v[71]); flagZeroHybrid = (int) parseDHRValue(v[49]); - if (flagZeroHybrid != 0 && flagZeroHybrid != 1) + if (flagZeroHybrid != 0 && flagZeroHybrid != 1) { flagZeroHybrid = 0; // should print warning + } } else if (s.equals("SUPL(13)")) { biasCalculated = parseDHRValue(v[69]); } @@ -521,23 +555,33 @@ public class RadarRecordUtil { } else if (s.equals("ADAP(46)")) { map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9])); map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10])); - map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25])); + map.put(DHRValues.MAXPRECIPRATEALLOW, + parseDHRValue(v[vi + 25])); s = v[68]; if (s.equals("BIAS(11)")) { - map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53])); biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083 + map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53])); + biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR + // 13083 biasCalculated = parseDHRValue(v[77]); } else if (s.equals("BIAS( 9)")) { - map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53])); biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR 13083 + map.put(DHRValues.BIASAPPLIEDFLAG, parseDHRValue(v[53])); + biasApplied = map.get(DHRValues.BIASAPPLIEDFLAG) > 0;// DR + // 13083 biasCalculated = parseDHRValue(v[73]); } vi = nv; } } - if (flagZeroHybrid != null) + if (flagZeroHybrid != null) { map.put(DHRValues.FLAGZEROHYBRID, (double) flagZeroHybrid); + } if (!biasApplied) { biasCalculated = 1.0; - } else { if(biasCalculated < 0.01 || biasCalculated > 100.0) biasCalculated = 1.0; } // DR 13083 + } else { + if (biasCalculated < 0.01 || biasCalculated > 100.0) { + biasCalculated = 1.0; + } + } // DR 13083 map.put(DHRValues.BIAS, biasCalculated); // Also include logic from A1 FFMPContainer::read(), FFMP_ORPG case @@ -555,44 +599,49 @@ public class RadarRecordUtil { private static double parseDHRValue(String text) { String s = text.trim(); - if (s.equals("T")) + if (s.equals("T")) { return 1; - else if (s.equals("F")) + } else if (s.equals("F")) { return 0; - else + } else { return Double.parseDouble(s); + } } -/** + /** * DR#11705: SCAN missing row(s) comparing to radar Comb Att Table. - * - * Error cause: RadarConstants.GRAPHIC_BLOCK as a Regular Expression - * pattern can not match some variations in Graphic Block Texts with - * "<" and ">" having spaces between them and their associated numbers - * ( MXHAILSIZE and TOP ). - * + * + * Error cause: RadarConstants.GRAPHIC_BLOCK as a Regular Expression pattern + * can not match some variations in Graphic Block Texts with "<" and ">" + * having spaces between them and their associated numbers ( MXHAILSIZE and + * TOP ). + * * Fix: replace all "<" and ">" with space: " " - * - * @param : Graphic Block Text that may contain ">" and/or "<". - * @return: String with ">" and/or "<" replaced by space. + * + * @param : + * Graphic Block Text that may contain ">" and/or "<". + * @return: String with ">" and/or "<" replaced by space. */ private static String getNormalizedGBText(String text) { if (text == null || text.isEmpty() - || ((!text.contains(">")) && (!text.contains("<")))) + || !text.contains(">") && !text.contains("<")) { return text; + } /* * contains only ">" */ - if (!text.contains("<")) + if (!text.contains("<")) { return text.replaceAll(">", " "); + } /* * contains only "<" */ - if (!text.contains(">")) + if (!text.contains(">")) { return text.replaceAll("<", " "); + } /* * contains both "<" and ">" @@ -600,4 +649,88 @@ public class RadarRecordUtil { return text.replaceAll(">", " ").replaceAll("<", " "); } + + /** + * Add x and y offset needed to bring the radar coordinate system into + * something geotools can handle + */ + public static Coordinate rectifyCoordinate(Coordinate c) { + c.x += 2048; + c.y += 2048; + c.y = 4096 - c.y; + return c; + } + + /** + * Builds the necessary coordinates for use in the wireframe shapes + * + * @param radarRecord + * @return + */ + public static Map buildMeltingLayerCoordinates( + RadarRecord radarRecord) { + SymbologyBlock block = radarRecord.getSymbologyBlock(); + GeneralGridGeometry gg = getRadarGraphicsGridGeometry(radarRecord); + ReferencedCoordinate coordinate; + Map coordinates = new HashMap<>(); + if (block != null) { + for (int i = 0; i < block.getNumLayers(); i++) { + for (int j = 1; j < block.getNumPackets(i); j++) { + if (block.getPacket(i, + j) instanceof LinkedContourVectorPacket) { + List vector = ((LinkedContourVectorPacket) block + .getPacket(i, j)).getVectors(); + Coordinate[] coords = new Coordinate[vector.size() + 1]; + for (int l = 0; l < coords.length - 1; l++) { + if (!coordinates + .containsKey(vector.get(l).getTheColor())) { + coordinates.put( + Integer.valueOf( + vector.get(l).getTheColor()), + coords); + } + + // transform the coordinates to the correct + // locations + coordinate = new ReferencedCoordinate( + rectifyCoordinate(new Coordinate( + vector.get(l).getI2(), + vector.get(l).getJ2())), + gg, Type.GRID_CENTER); + try { + coords[l] = coordinate.asLatLon(); + } catch (TransformException | FactoryException e) { + statusHandler.handle(Priority.PROBLEM, + e.getLocalizedMessage(), e); + } + } + coords[coords.length - 1] = coords[0]; + } + } + } + } + return coordinates; + } + + /** + * @param radarRecord + * A radar graphics record + * @return Grid geometry suitable for radar graphics only. Not for other + * types of radar products. + */ + public static GeneralGridGeometry getRadarGraphicsGridGeometry( + RadarRecord radarRecord) { + GeneralEnvelope generalEnvelope = new GeneralEnvelope(2); + // Per section 3.3.3 + generalEnvelope.setCoordinateReferenceSystem(radarRecord.getCRS()); + generalEnvelope.setRange(0, -256000 * 2, 256000 * 2); + generalEnvelope.setRange(1, -256000 * 2, 256000 * 2); + // [-2048, 2048] == range of 4095 (inclusive 0), plus 1 + // because GGR is exclusive (?) + GeneralGridGeometry rval = new GeneralGridGeometry( + new GeneralGridEnvelope(new int[] { 0, 0 }, + new int[] { 4096, 4096 }, false), + generalEnvelope); + return rval; + } } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarTextProductUtil.java b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarTextProductUtil.java index eb97d348df..d1fcba9402 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarTextProductUtil.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarTextProductUtil.java @@ -36,6 +36,7 @@ import java.util.List; * dependency * Apr 14, 2016 DR18800 jdynina Removed alerting * Apr 15, 2016 DR18796 jdynina Added SCC + * Mar 03, 2017 DR19848 jdynina Changed SCC to SCL to prevent conflicts * * * @@ -80,7 +81,7 @@ public class RadarTextProductUtil { put(150, "WSRUSW"); put(151, "WSRUSD"); put(171, "WSRSTA"); - put(202, "WSRSCC"); + put(202, "WSRSCL"); } }; diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/dataaccess/SatelliteGridFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/dataaccess/SatelliteGridFactory.java index 22884e4abd..994159107f 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/dataaccess/SatelliteGridFactory.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.satellite/src/com/raytheon/uf/common/dataplugin/satellite/dataaccess/SatelliteGridFactory.java @@ -68,6 +68,7 @@ import com.raytheon.uf.common.numeric.source.DataSource; * IDataRequest * Jun 07, 2016 5574 tgurney Add advanced query support * Aug 01, 2016 2416 tgurney Add dataURI as optional identifier + * Mar 06, 2017 6142 bsteffen Remove dataURI as optional identifier * * * @@ -84,8 +85,7 @@ public class SatelliteGridFactory extends AbstractGridDataPluginFactory { private static final String FIELD_SOURCE = "source"; private static final String[] OPTIONAL_IDENTIFIERS = { FIELD_SOURCE, - FIELD_CREATING_ENTITY, FIELD_SECTOR_ID, FIELD_PHYSICAL_ELEMENT, - PluginDataObject.DATAURI_ID }; + FIELD_CREATING_ENTITY, FIELD_SECTOR_ID, FIELD_PHYSICAL_ELEMENT }; public SatelliteGridFactory() { SatelliteUnits.register(); @@ -199,16 +199,17 @@ public class SatelliteGridFactory extends AbstractGridDataPluginFactory { */ @Override public String[] getAvailableParameters(IDataRequest request) { - return getAvailableValues(request, FIELD_PHYSICAL_ELEMENT, String.class); + return getAvailableValues(request, FIELD_PHYSICAL_ELEMENT, + String.class); } @Override public String[] getIdentifierValues(IDataRequest request, String identifierKey) { - if (!Arrays.asList(getRequiredIdentifiers(request)).contains( - identifierKey) - && !Arrays.asList(getOptionalIdentifiers(request)).contains( - identifierKey)) { + if (!Arrays.asList(getRequiredIdentifiers(request)) + .contains(identifierKey) + && !Arrays.asList(getOptionalIdentifiers(request)) + .contains(identifierKey)) { throw new InvalidIdentifiersException(request.getDatatype(), null, Arrays.asList(new String[] { identifierKey })); } diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.dataaccess/META-INF/MANIFEST.MF index b808019811..2f3838b127 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Data Access Framework EDEX Bundle-SymbolicName: com.raytheon.uf.edex.dataaccess -Bundle-Version: 1.14.0.qualifier +Bundle-Version: 1.16.0.qualifier Bundle-Vendor: RAYTHEON Require-Bundle: com.raytheon.uf.common.dataaccess, com.raytheon.uf.common.serialization.comm, diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/res/spring/dataaccess-request.xml b/edexOsgi/com.raytheon.uf.edex.dataaccess/res/spring/dataaccess-request.xml index 71e6bb0609..899b0b6855 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/res/spring/dataaccess-request.xml +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/res/spring/dataaccess-request.xml @@ -2,92 +2,29 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd"> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - + diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridDataHandler.java b/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridDataHandler.java index 1ce5055a7d..335d8bb00e 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridDataHandler.java +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridDataHandler.java @@ -39,24 +39,16 @@ import com.raytheon.uf.common.util.CollectionUtil; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Jun 4, 2013 dgilling Initial creation + * Jun 4, 2013 dgilling Initial creation + * Oct 18, 2016 5916 bsteffen Allow lazy loading of lat/lon data * * * * @author dgilling - * @version 1.0 */ - public final class GetGridDataHandler implements IRequestHandler { - /* - * (non-Javadoc) - * - * @see - * com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest - * (com.raytheon.uf.common.serialization.comm.IServerRequest) - */ @Override public GetGridDataResponse handleRequest(final GetGridDataRequest request) throws Exception { @@ -73,7 +65,7 @@ public final class GetGridDataHandler implements } GetGridDataResponse response = new GetGridDataResponse( - Arrays.asList(gridData)); + Arrays.asList(gridData), request.isIncludeLatLonData()); return response; } } diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridLatLonHandler.java b/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridLatLonHandler.java new file mode 100644 index 0000000000..502e72e37a --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/src/com/raytheon/uf/edex/dataaccess/handler/GetGridLatLonHandler.java @@ -0,0 +1,70 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.dataaccess.handler; + +import org.geotools.coverage.grid.GridEnvelope2D; +import org.geotools.coverage.grid.GridGeometry2D; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.CRS; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.FactoryException; + +import com.raytheon.uf.common.dataaccess.request.GetGridLatLonRequest; +import com.raytheon.uf.common.dataaccess.response.GetGridLatLonResponse; +import com.raytheon.uf.common.geospatial.LatLonReprojection; +import com.raytheon.uf.common.geospatial.LatLonWrapper; +import com.raytheon.uf.common.serialization.comm.IRequestHandler; + +/** + * + * Handler for {@link GetGridLatLonRequest}. + * + * 
+ *
+ * SOFTWARE HISTORY
+ * 
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- -----------------
+ * Oct 18, 2016  5916     bsteffen  Initial creation
+ * 
+ *
+ * + * @author bsteffen + */ +public final class GetGridLatLonHandler + implements IRequestHandler { + + @Override + public GetGridLatLonResponse handleRequest(GetGridLatLonRequest request) + throws MismatchedDimensionException, FactoryException { + GridEnvelope2D range = new GridEnvelope2D(0, 0, request.getNx(), + request.getNy()); + ReferencedEnvelope envelope = new ReferencedEnvelope( + request.getEnvelope(), CRS.parseWKT(request.getCrsWkt())); + GridGeometry2D gridGeometry = new GridGeometry2D(range, envelope); + GetGridLatLonResponse response = new GetGridLatLonResponse(); + LatLonWrapper latLonData = LatLonReprojection.getLatLons(gridGeometry); + response.setNx(request.getNx()); + response.setNy(request.getNy()); + response.setLats(latLonData.getLats()); + response.setLons(latLonData.getLons()); + return response; + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py index 6f5c146756..41d9229ba0 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JData.py @@ -18,7 +18,6 @@ # further licensing information. ## - # # Implements IData and wraps around a Java IData # @@ -32,6 +31,12 @@ # # +## +# This is a base file that is not intended to be overridden. +## + + + from awips.dataaccess import IData import JUtil, DataTime diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py index c2602ad26c..a4dddd23c4 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JDataRequest.py @@ -18,7 +18,6 @@ # further licensing information. ## - # # Implements IDataRequest and wraps around a Java IDataRequest # @@ -36,6 +35,12 @@ # # +## +# This is a base file that is not intended to be overridden. +## + + + from awips.dataaccess import IDataRequest from com.raytheon.uf.common.dataplugin.level import Level import JUtil diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py index 2dc47a1f94..4d5a9b1713 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGeometryData.py @@ -18,7 +18,6 @@ # further licensing information. ## - # # Implements IGeometryData and wraps around a Java IGeometryData. # @@ -34,6 +33,12 @@ # # +## +# This is a base file that is not intended to be overridden. +## + + + from awips.dataaccess import IGeometryData import JData diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py index f989cdd803..3286ed5d15 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JGridData.py @@ -18,7 +18,6 @@ # further licensing information. # # - # # Implements IGridData and wraps around a Java IGridData. # @@ -32,10 +31,15 @@ # 08/06/14 3185 njensen Only import Java classes when necessary # Apr 23, 2015 4259 njensen Updated for new JEP API # 11/02/15 5079 dgilling Fix typo in getRawData. -# 11/10/16 5900 bsteffen Correct grid shape, return longitude -# -# +# 10/14/16 5916 bsteffen Correct grid shape, return longitude # +# + +## +# This is a base file that is not intended to be overridden. +## + + from awips.dataaccess import IGridData import JData diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JepRouter.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JepRouter.py index 5d070be8fb..af191296f5 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JepRouter.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/JepRouter.py @@ -18,7 +18,6 @@ # further licensing information. # # - # # Routes requests to the Data Access Framework through JEP to the Java classes. # Returns Python objects that wrap Java objects. @@ -43,7 +42,12 @@ # getOptionalIdentifiers() # # -# + +## +# This is a base file that is not intended to be overridden. +## + + from awips.dataaccess import IDataRequest diff --git a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/__init__.py b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/__init__.py index 2953071052..9601d1baeb 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/__init__.py +++ b/edexOsgi/com.raytheon.uf.edex.dataaccess/utility/common_static/base/python/dataaccess/__init__.py @@ -18,7 +18,6 @@ # further licensing information. ## - # # __init__.py for dataaccess python that is used within a JVM. # @@ -28,9 +27,13 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 12/10/12 njensen Initial Creation. -# -# # +# + +## +# This is a base file that is not intended to be overridden. +## + __all__ = [ diff --git a/pythonPackages/dynamicserialize/ThriftSerializationContext.py b/pythonPackages/dynamicserialize/ThriftSerializationContext.py index 071dc55cba..e82e2118d3 100644 --- a/pythonPackages/dynamicserialize/ThriftSerializationContext.py +++ b/pythonPackages/dynamicserialize/ThriftSerializationContext.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -27,39 +27,46 @@ # languages, it is instead all based on inspecting the types of the objects # passed to it. Therefore, ensure the types of python objects and primitives # match what they should be in the destination language. -# -# -# SOFTWARE HISTORY -# +# +# +# SOFTWARE HISTORY +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 06/09/10 njensen Initial Creation. # 06/12/13 #2099 dgilling Implement readObject() and # writeObject(). # Apr 24, 2015 4425 nabowle Add Double support +# Oct 17, 2016 5919 njensen Optimized for speed # # from thrift.Thrift import TType -import inspect, sys, types +import inspect +import sys +import types +import time import dynamicserialize from dynamicserialize import dstypes, adapters import SelfDescribingBinaryProtocol import numpy +DS_LEN = len('dynamicserialize.dstypes.') + dsObjTypes = {} - + + def buildObjMap(module): - if module.__dict__.has_key('__all__'): + if '__all__' in module.__dict__: for i in module.__all__: name = module.__name__ + '.' + i __import__(name) buildObjMap(sys.modules[name]) - else: + else: clzName = module.__name__[module.__name__.rfind('.') + 1:] clz = module.__dict__[clzName] tname = module.__name__ - tname = tname.replace('dynamicserialize.dstypes.', '') + tname = tname[DS_LEN:] dsObjTypes[tname] = clz buildObjMap(dstypes) @@ -72,7 +79,7 @@ pythonToThriftMap = { types.DictionaryType: TType.MAP, type(set([])): TType.SET, types.FloatType: SelfDescribingBinaryProtocol.FLOAT, - #types.FloatType: TType.DOUBLE, + # types.FloatType: TType.DOUBLE, types.BooleanType: TType.BOOL, types.InstanceType: TType.STRUCT, types.NoneType: TType.VOID, @@ -87,126 +94,122 @@ pythonToThriftMap = { numpy.int64: TType.I64 } -primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) +primitiveSupport = (TType.BYTE, TType.I16, TType.I32, TType.I64, + SelfDescribingBinaryProtocol.FLOAT, TType.DOUBLE) + class ThriftSerializationContext(object): - + def __init__(self, serializationManager, selfDescribingBinaryProtocol): self.serializationManager = serializationManager self.protocol = selfDescribingBinaryProtocol self.typeDeserializationMethod = { - TType.STRING: self.protocol.readString, - TType.I16: self.protocol.readI16, - TType.I32: self.protocol.readI32, - TType.LIST: self._deserializeArray, - TType.MAP: self._deserializeMap, - TType.SET: self._deserializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, - TType.BYTE: self.protocol.readByte, - TType.I64: self.protocol.readI64, - TType.DOUBLE: self.protocol.readDouble, - TType.BOOL: self.protocol.readBool, - TType.STRUCT: self.deserializeMessage, - TType.VOID: lambda: None - } + TType.STRING: self.protocol.readString, + TType.I16: self.protocol.readI16, + TType.I32: self.protocol.readI32, + TType.LIST: self._deserializeArray, + TType.MAP: self._deserializeMap, + TType.SET: self._deserializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readFloat, + TType.BYTE: self.protocol.readByte, + TType.I64: self.protocol.readI64, + TType.DOUBLE: self.protocol.readDouble, + TType.BOOL: self.protocol.readBool, + TType.STRUCT: self.deserializeMessage, + TType.VOID: lambda: None + } self.typeSerializationMethod = { - TType.STRING: self.protocol.writeString, - TType.I16: self.protocol.writeI16, - TType.I32: self.protocol.writeI32, - TType.LIST: self._serializeArray, - TType.MAP: self._serializeMap, - TType.SET: self._serializeSet, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, - TType.BYTE: self.protocol.writeByte, - TType.I64: self.protocol.writeI64, - TType.DOUBLE: self.protocol.writeDouble, - TType.BOOL: self.protocol.writeBool, - TType.STRUCT: self.serializeMessage, - TType.VOID: lambda x: None - } + TType.STRING: self.protocol.writeString, + TType.I16: self.protocol.writeI16, + TType.I32: self.protocol.writeI32, + TType.LIST: self._serializeArray, + TType.MAP: self._serializeMap, + TType.SET: self._serializeSet, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeFloat, + TType.BYTE: self.protocol.writeByte, + TType.I64: self.protocol.writeI64, + TType.DOUBLE: self.protocol.writeDouble, + TType.BOOL: self.protocol.writeBool, + TType.STRUCT: self.serializeMessage, + TType.VOID: lambda x: None + } self.listDeserializationMethod = { - TType.BYTE: self.protocol.readI8List, - TType.I16: self.protocol.readI16List, - TType.I32: self.protocol.readI32List, - TType.I64: self.protocol.readI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, - TType.DOUBLE: self.protocol.readF64List - } + TType.BYTE: self.protocol.readI8List, + TType.I16: self.protocol.readI16List, + TType.I32: self.protocol.readI32List, + TType.I64: self.protocol.readI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.readF32List, + TType.DOUBLE: self.protocol.readF64List + } self.listSerializationMethod = { - TType.BYTE: self.protocol.writeI8List, - TType.I16: self.protocol.writeI16List, - TType.I32: self.protocol.writeI32List, - TType.I64: self.protocol.writeI64List, - SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, - TType.DOUBLE: self.protocol.writeF64List - } - - + TType.BYTE: self.protocol.writeI8List, + TType.I16: self.protocol.writeI16List, + TType.I32: self.protocol.writeI32List, + TType.I64: self.protocol.writeI64List, + SelfDescribingBinaryProtocol.FLOAT: self.protocol.writeF32List, + TType.DOUBLE: self.protocol.writeF64List + } + def readMessageStart(self): - msg = self.protocol.readMessageBegin() + msg = self.protocol.readMessageBegin() return msg[0] - + def readMessageEnd(self): self.protocol.readMessageEnd() - + def deserializeMessage(self): name = self.protocol.readStructBegin() - name = name.replace('_', '.') if name.isdigit(): obj = self._deserializeType(int(name)) return obj - elif adapters.classAdapterRegistry.has_key(name): + name = name.replace('_', '.') + if name in adapters.classAdapterRegistry: return adapters.classAdapterRegistry[name].deserialize(self) - elif name.find('$') > -1: - # it's an inner class, we're going to hope it's an enum, treat it special - fieldName, fieldType, fieldId = self.protocol.readFieldBegin() + elif '$' in name: + # it's an inner class, we're going to hope it's an enum, treat it + # special + fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldName != '__enumValue__': - raise dynamiceserialize.SerializationException("Expected to find enum payload. Found: " + fieldName) - obj = self.protocol.readString() + raise dynamiceserialize.SerializationException( + "Expected to find enum payload. Found: " + fieldName) + obj = self.protocol.readString() self.protocol.readFieldEnd() return obj - else: + else: clz = dsObjTypes[name] obj = clz() - + while self._deserializeField(name, obj): pass - + self.protocol.readStructEnd() return obj - + def _deserializeType(self, b): - if self.typeDeserializationMethod.has_key(b): + try: return self.typeDeserializationMethod[b]() - else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(b)) - - + except KeyError: + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(b)) + def _deserializeField(self, structname, obj): fieldName, fieldType, fieldId = self.protocol.readFieldBegin() if fieldType == TType.STOP: return False elif fieldType != TType.VOID: -# if adapters.fieldAdapterRegistry.has_key(structname) and adapters.fieldAdapterRegistry[structname].has_key(fieldName): -# result = adapters.fieldAdapterRegistry[structname][fieldName].deserialize(self) -# else: result = self._deserializeType(fieldType) lookingFor = "set" + fieldName[0].upper() + fieldName[1:] try: setMethod = getattr(obj, lookingFor) - - if callable(setMethod): - setMethod(result) - else: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) + setMethod(result) except: - raise dynamicserialize.SerializationException("Couldn't find setter method " + lookingFor) - + raise dynamicserialize.SerializationException( + "Couldn't find setter method " + lookingFor) + self.protocol.readFieldEnd() return True - - + def _deserializeArray(self): listType, size = self.protocol.readListBegin() result = [] @@ -218,42 +221,43 @@ class ThriftSerializationContext(object): result = self.listDeserializationMethod[listType](size) self.protocol.readListEnd() return result - + def _deserializeMap(self): keyType, valueType, size = self.protocol.readMapBegin() - result = {} + result = {} for n in xrange(size): # can't go off the type, due to java generics limitations dynamic serialize is # serializing keys and values as void key = self.typeDeserializationMethod[TType.STRUCT]() value = self.typeDeserializationMethod[TType.STRUCT]() - result[key] = value + result[key] = value self.protocol.readMapEnd() return result - + def _deserializeSet(self): - setType, setSize = self.protocol.readSetBegin() + setType, setSize = self.protocol.readSetBegin() result = set([]) for n in xrange(setSize): result.add(self.typeDeserializationMethod[TType.STRUCT]()) self.protocol.readSetEnd() return result - - def _lookupType(self, obj): + + def _lookupType(self, obj): pyt = type(obj) - if pythonToThriftMap.has_key(pyt): + if pyt in pythonToThriftMap: return pythonToThriftMap[pyt] - elif pyt.__module__.startswith('dynamicserialize.dstypes'): + elif pyt.__module__[:DS_LEN - 1] == ('dynamicserialize.dstypes'): return pythonToThriftMap[types.InstanceType] - else: - raise dynamicserialize.SerializationException("Don't know how to serialize object of type: " + str(pyt)) - + else: + raise dynamicserialize.SerializationException( + "Don't know how to serialize object of type: " + str(pyt)) + def serializeMessage(self, obj): tt = self._lookupType(obj) - + if tt == TType.STRUCT: - fqn = obj.__module__.replace('dynamicserialize.dstypes.', '') - if adapters.classAdapterRegistry.has_key(fqn): + fqn = obj.__module__[DS_LEN:] + if fqn in adapters.classAdapterRegistry: # get proper class name when writing class name to serialization stream # in case we have a special inner-class case m = sys.modules[adapters.classAdapterRegistry[fqn].__name__] @@ -264,7 +268,7 @@ class ThriftSerializationContext(object): return else: self.protocol.writeStructBegin(fqn) - methods = inspect.getmembers(obj, inspect.ismethod) + methods = inspect.getmembers(obj, inspect.ismethod) fid = 1 for m in methods: methodName = m[0] @@ -273,55 +277,56 @@ class ThriftSerializationContext(object): val = m[1]() ft = self._lookupType(val) if ft == TType.STRUCT: - fc = val.__module__.replace('dynamicserialize.dstypes.', '') + fc = val.__module__[DS_LEN:] self._serializeField(fieldname, ft, fid, val) else: self._serializeField(fieldname, ft, fid, val) fid += 1 self.protocol.writeFieldStop() - + self.protocol.writeStructEnd() else: # basic types self.protocol.writeStructBegin(str(tt)) self._serializeType(obj, tt) self.protocol.writeStructEnd() - + def _serializeField(self, fieldName, fieldType, fieldId, fieldValue): self.protocol.writeFieldBegin(fieldName, fieldType, fieldId) self._serializeType(fieldValue, fieldType) self.protocol.writeFieldEnd() - + def _serializeType(self, fieldValue, fieldType): - if self.typeSerializationMethod.has_key(fieldType): + if fieldType in self.typeSerializationMethod: return self.typeSerializationMethod[fieldType](fieldValue) else: - raise dynamicserialize.SerializationException("Unsupported type value " + str(fieldType)) - + raise dynamicserialize.SerializationException( + "Unsupported type value " + str(fieldType)) + def _serializeArray(self, obj): size = len(obj) if size: if type(obj) is numpy.ndarray: - t = pythonToThriftMap[obj.dtype.type] - size = obj.size + t = pythonToThriftMap[obj.dtype.type] + size = obj.size else: t = self._lookupType(obj[0]) else: - t = TType.STRUCT + t = TType.STRUCT self.protocol.writeListBegin(t, size) if t == TType.STRING: if type(obj) is numpy.ndarray: if len(obj.shape) == 1: - for x in obj: + for x in obj: s = str(x).strip() self.typeSerializationMethod[t](s) - else: + else: for x in obj: for y in x: s = str(y).strip() self.typeSerializationMethod[t](s) else: - for x in obj: + for x in obj: s = str(x) self.typeSerializationMethod[t](s) elif t not in primitiveSupport: @@ -330,8 +335,7 @@ class ThriftSerializationContext(object): else: self.listSerializationMethod[t](obj) self.protocol.writeListEnd() - - + def _serializeMap(self, obj): size = len(obj) self.protocol.writeMapBegin(TType.VOID, TType.VOID, size) @@ -339,83 +343,82 @@ class ThriftSerializationContext(object): self.typeSerializationMethod[TType.STRUCT](k) self.typeSerializationMethod[TType.STRUCT](obj[k]) self.protocol.writeMapEnd() - + def _serializeSet(self, obj): size = len(obj) self.protocol.writeSetBegin(TType.VOID, size) for x in obj: self.typeSerializationMethod[TType.STRUCT](x) self.protocol.writeSetEnd() - + def writeMessageStart(self, name): self.protocol.writeMessageBegin(name, TType.VOID, 0) - + def writeMessageEnd(self): self.protocol.writeMessageEnd() - + def readBool(self): return self.protocol.readBool() - + def writeBool(self, b): self.protocol.writeBool(b) - + def readByte(self): return self.protocol.readByte() - + def writeByte(self, b): self.protocol.writeByte(b) - + def readDouble(self): return self.protocol.readDouble() - + def writeDouble(self, d): self.protocol.writeDouble(d) - + def readFloat(self): return self.protocol.readFloat() - + def writeFloat(self, f): self.protocol.writeFloat(f) - + def readI16(self): return self.protocol.readI16() - + def writeI16(self, i): self.protocol.writeI16(i) - + def readI32(self): return self.protocol.readI32() - + def writeI32(self, i): self.protocol.writeI32(i) - + def readI64(self): return self.protocol.readI64() - + def writeI64(self, i): self.protocol.writeI64(i) - + def readString(self): return self.protocol.readString() - + def writeString(self, s): self.protocol.writeString(s) - + def readBinary(self): numBytes = self.protocol.readI32() return self.protocol.readI8List(numBytes) - + def readFloatArray(self): size = self.protocol.readI32() return self.protocol.readF32List(size) - + def writeFloatArray(self, floats): self.protocol.writeI32(len(floats)) self.protocol.writeF32List(floats) - + def readObject(self): return self.deserializeMessage() - + def writeObject(self, obj): self.serializeMessage(obj) - \ No newline at end of file diff --git a/pythonPackages/dynamicserialize/adapters/__init__.py b/pythonPackages/dynamicserialize/adapters/__init__.py index d7e643fa2d..fa6bdc954b 100644 --- a/pythonPackages/dynamicserialize/adapters/__init__.py +++ b/pythonPackages/dynamicserialize/adapters/__init__.py @@ -1,19 +1,19 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## @@ -21,56 +21,80 @@ # # __init__.py for Dynamic Serialize adapters. -# -# +# +# Plugins can contribute to dynamicserialize.adapters by either including their +# classes directly in pythonPackages/dynamicserialize/adapters/ within their +# plugin. The plugin's adapter will automatically be added to __all__ at runtime +# and registered. +# Plugins should not include a custom __init__.py in +# pythonPackages/dynamicserialize/adapters/ because it will overwrite this file. +# If custom package initialization is needed, a subpackage should be created +# with an __init__.py that includes the following: +# +# __all__ = ['CustomAdapter1', 'CustomAdapter2'] +# from dynamicserialize.adapters import registerAdapters +# registerAdapters(__name__, __all__) +# +# # SOFTWARE HISTORY -# +# # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- -# 08/31/10 njensen Initial Creation. -# 03/20/13 #1774 randerso Added TimeConstraintsAdapter -# 04/22/13 #1949 rjpeter Added LockTableAdapter -# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter -# 06/22/2015 #4573 randerso Added JobProgressAdapter -# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter -# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 08/31/10 njensen Initial Creation. +# 03/20/13 #1774 randerso Added TimeConstraintsAdapter +# 04/22/13 #1949 rjpeter Added LockTableAdapter +# 02/06/14 #2672 bsteffen Added JTSEnvelopeAdapter +# 06/22/2015 #4573 randerso Added JobProgressAdapter +# 09/21/2015 #4486 rjpeter Added FormattedDateAdapter +# 06/23/2016 #5696 rjpeter Added CommutativeTimestampAdapter +# 10/17/2016 #5919 njensen Added GeomDataRespAdapter +# 01/09/2017 #5997 nabowle Allow contribution from plugins. # __all__ = [ - 'PointAdapter', - 'StackTraceElementAdapter', - 'WsIdAdapter', - 'CalendarAdapter', - 'GregorianCalendarAdapter', - 'ActiveTableModeAdapter', - 'DateAdapter', - 'FormattedDateAdapter', - 'LocalizationLevelSerializationAdapter', - 'LocalizationTypeSerializationAdapter', - 'GeometryTypeAdapter', - 'CoordAdapter', - 'TimeRangeTypeAdapter', - 'ParmIDAdapter', - 'DatabaseIDAdapter', - 'TimestampAdapter', - 'CommutativeTimestampAdapter', - 'EnumSetAdapter', - 'FloatBufferAdapter', - 'ByteBufferAdapter', - 'TimeConstraintsAdapter', - 'LockTableAdapter', - 'JTSEnvelopeAdapter', - 'JobProgressAdapter', - ] - + 'PointAdapter', + 'StackTraceElementAdapter', + 'CalendarAdapter', + 'GregorianCalendarAdapter', + 'DateAdapter', + 'GeometryTypeAdapter', + 'CoordAdapter', + 'TimestampAdapter', + 'EnumSetAdapter', + 'FloatBufferAdapter', + 'ByteBufferAdapter', + 'JTSEnvelopeAdapter' +] + classAdapterRegistry = {} - + def getAdapterRegistry(): - import sys - for x in __all__: - exec 'import ' + x - m = sys.modules['dynamicserialize.adapters.' + x] + import pkgutil + + discoveredPackages = [] + # allow other plugins to contribute to adapters by dropping their adapter or + # package into the dynamicserialize.adapters package + for _, modname, ispkg in pkgutil.iter_modules(__path__): + if ispkg: + discoveredPackages.append(modname) + else: + if modname not in __all__: + __all__.append(modname) + + registerAdapters(__name__, __all__) + + for pkg in discoveredPackages: + __import__(__name__ + '.' + pkg) + + +def registerAdapters(package, modules): + import sys + if not package.endswith('.'): + package += '.' + for x in modules: + exec 'import ' + package + x + m = sys.modules[package + x] d = m.__dict__ if d.has_key('ClassAdapter'): if isinstance(m.ClassAdapter, list): @@ -80,9 +104,8 @@ def getAdapterRegistry(): clzName = m.ClassAdapter classAdapterRegistry[clzName] = m else: - raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + \ + raise LookupError('Adapter class ' + x + ' has no ClassAdapter field ' + 'and cannot be registered.') - + getAdapterRegistry() - diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py index 96fc3e32c9..966f28b25c 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/__init__.py @@ -21,22 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'activetable', - 'alertviz', - 'auth', - 'dataaccess', - 'dataplugin', - 'dataquery', - 'datastorage', - 'localization', - 'management', - 'message', - 'plugin', - 'pointdata', - 'pypies', - 'serialization', - 'site', - 'time' + 'dataplugin' ] diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py index 4bb023b6f0..c18ecc8342 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/__init__.py @@ -21,13 +21,7 @@ # File auto-generated by PythonFileGenerator __all__ = [ - 'events', - 'gfe', - 'grid', - 'level', - 'message', - 'radar', - 'text' + 'events' ] diff --git a/pythonPackages/ufpy/NotificationMessage.py b/pythonPackages/ufpy/NotificationMessage.py index 92036e49d4..bf22c738a5 100755 --- a/pythonPackages/ufpy/NotificationMessage.py +++ b/pythonPackages/ufpy/NotificationMessage.py @@ -42,7 +42,7 @@ from dynamicserialize import DynamicSerializationManager # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 09/30/08 chammack Initial Creation. -# 11/03/10 5849 cjeanbap Moved to awips package from +# 11/03/10 5849 cjeanbap Moved to ufpy package from # com.raytheon.uf.tools.cli # 01/07/11 5645 cjeanbap Added audio file to Status Message. # 05/27/11 3050 cjeanbap Added if-statement to check Priority diff --git a/pythonPackages/ufpy/QpidSubscriber.py b/pythonPackages/ufpy/QpidSubscriber.py index 66ec0c522b..7ea5be0195 100644 --- a/pythonPackages/ufpy/QpidSubscriber.py +++ b/pythonPackages/ufpy/QpidSubscriber.py @@ -30,9 +30,12 @@ # 11/17/10 njensen Initial Creation. # 08/15/13 2169 bkowal Optionally gzip decompress any data that is read. # 08/04/16 2416 tgurney Add queueStarted property +# 02/16/17 6084 bsteffen Support ssl connections # # +import os +import os.path import qpid import zlib @@ -41,11 +44,24 @@ from qpid.exceptions import Closed class QpidSubscriber: - def __init__(self, host='127.0.0.1', port=5672, decompress=False): + def __init__(self, host='127.0.0.1', port=5672, decompress=False, ssl=None): self.host = host self.port = port self.decompress = decompress; socket = qpid.util.connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = "guest" + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) self.__connection = qpid.connection.Connection(sock=socket, username='guest', password='guest') self.__connection.start() self.__session = self.__connection.session(str(qpid.datatypes.uuid4())) @@ -103,4 +119,4 @@ class QpidSubscriber: @property def queueStarted(self): return self.__queueStarted - \ No newline at end of file + diff --git a/pythonPackages/ufpy/UsageArgumentParser.py b/pythonPackages/ufpy/UsageArgumentParser.py index 3d51564a3f..313d206216 100644 --- a/pythonPackages/ufpy/UsageArgumentParser.py +++ b/pythonPackages/ufpy/UsageArgumentParser.py @@ -1,29 +1,39 @@ ## # This software was developed and / or modified by Raytheon Company, # pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# +# # U.S. EXPORT CONTROLLED TECHNICAL DATA # This software product contains export-restricted data whose # export/transfer/disclosure is restricted by U.S. law. Dissemination # to non-U.S. persons whether in the United States or abroad requires # an export license or other authorization. -# +# # Contractor Name: Raytheon Company # Contractor Address: 6825 Pine Street, Suite 340 # Mail Stop B8 # Omaha, NE 68106 # 402.291.0100 -# +# # See the AWIPS II Master Rights File ("Master Rights File.pdf") for # further licensing information. ## +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------- -------- --------- --------------------------------------------- +# Feb 13, 2017 6092 randerso Added StoreTimeAction +# +## import argparse import sys +import time from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID +TIME_FORMAT = "%Y%m%d_%H%M" class UsageArgumentParser(argparse.ArgumentParser): """ @@ -56,3 +66,16 @@ class AppendParmNameAndLevelAction(argparse.Action): else: setattr(namespace, self.dest, [comp]) +class StoreTimeAction(argparse.Action): + """ + argparse.Action subclass to validate GFE formatted time strings + and parse them to time.struct_time + """ + def __call__(self, parser, namespace, values, option_string=None): + try: + timeStruct = time.strptime(values, TIME_FORMAT) + except: + parser.error(str(values) + " is not a valid time string of the format YYYYMMDD_hhmm") + + setattr(namespace, self.dest, timeStruct) + diff --git a/pythonPackages/ufpy/__init__.py b/pythonPackages/ufpy/__init__.py index f1985c4f73..6ed9ac6af6 100644 --- a/pythonPackages/ufpy/__init__.py +++ b/pythonPackages/ufpy/__init__.py @@ -20,7 +20,7 @@ # -# __init__.py for awips package +# __init__.py for ufpy package # # # SOFTWARE HISTORY diff --git a/pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py b/pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py index 2956acda5c..fcf9ba4e90 100644 --- a/pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py +++ b/pythonPackages/ufpy/dataaccess/CombinedTimeQuery.py @@ -31,7 +31,7 @@ # 06/22/16 #5591 bsteffen Initial Creation. # -from awips.dataaccess import DataAccessLayer +from ufpy.dataaccess import DataAccessLayer def getAvailableTimes(request, refTimeOnly=False): return __getAvailableTimesForEachParameter(request, refTimeOnly) diff --git a/pythonPackages/ufpy/dataaccess/DataAccessLayer.py b/pythonPackages/ufpy/dataaccess/DataAccessLayer.py index 5a22ddf61b..868d871011 100644 --- a/pythonPackages/ufpy/dataaccess/DataAccessLayer.py +++ b/pythonPackages/ufpy/dataaccess/DataAccessLayer.py @@ -20,7 +20,7 @@ # -# Published interface for awips.dataaccess package +# Published interface for ufpy.dataaccess package # # # SOFTWARE HISTORY @@ -40,7 +40,7 @@ # Jun 01, 2016 5587 tgurney Add new signatures for # getRequiredIdentifiers() and # getOptionalIdentifiers() -# +# Oct 18, 2016 5916 bsteffen Add setLazyLoadGridLatLon # # @@ -64,7 +64,7 @@ if sys.modules.has_key('jep'): import JepRouter router = JepRouter else: - from awips.dataaccess import ThriftClientRouter + from ufpy.dataaccess import ThriftClientRouter router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) USING_NATIVE_THRIFT = True @@ -251,3 +251,26 @@ def changeEDEXHost(newHostName): router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) else: raise TypeError("Cannot call changeEDEXHost when using JepRouter.") + +def setLazyLoadGridLatLon(lazyLoadGridLatLon): + """ + Provide a hint to the Data Access Framework indicating whether to load the + lat/lon data for a grid immediately or wait until it is needed. This is + provided as a performance tuning hint and should not affect the way the + Data Access Framework is used. Depending on the internal implementation of + the Data Access Framework this hint might be ignored. Examples of when this + should be set to True are when the lat/lon information is not used or when + it is used only if certain conditions within the data are met. It could be + set to False if it is guaranteed that all lat/lon information is needed and + it would be better to get any performance overhead for generating the + lat/lon data out of the way during the initial request. + + + Args: + lazyLoadGridLatLon: Boolean value indicating whether to lazy load. + """ + try: + router.setLazyLoadGridLatLon(lazyLoadGridLatLon) + except AttributeError: + # The router is not required to support this capability. + pass \ No newline at end of file diff --git a/pythonPackages/ufpy/dataaccess/DataNotificationLayer.py b/pythonPackages/ufpy/dataaccess/DataNotificationLayer.py index f590803d42..5ec788d843 100644 --- a/pythonPackages/ufpy/dataaccess/DataNotificationLayer.py +++ b/pythonPackages/ufpy/dataaccess/DataNotificationLayer.py @@ -19,7 +19,7 @@ # # # -# Published interface for retrieving data updates via awips.dataaccess package +# Published interface for retrieving data updates via ufpy.dataaccess package # # # SOFTWARE HISTORY @@ -37,7 +37,7 @@ retrieval of new data as it is coming into the system. There are two ways to access this feature: -1. The DataQueue module (awips.dataaccess.DataQueue) offers a collection that +1. The DataQueue module (ufpy.dataaccess.DataQueue) offers a collection that automatically fills up with new data as it receives notifications. See that module for more information. @@ -49,8 +49,8 @@ each time new data is received. Example code follows. This example prints temperature as observed from KOMA each time a METAR is received from there. - from awips.dataaccess import DataAccessLayer as DAL - from awips.dataaccess import DataNotificationLayer as DNL + from ufpy.dataaccess import DataAccessLayer as DAL + from ufpy.dataaccess import DataNotificationLayer as DNL def process_obs(list_of_data): for item in list_of_data: @@ -69,8 +69,8 @@ each time a METAR is received from there. import re import sys import subprocess -from awips.dataaccess.PyGeometryNotification import PyGeometryNotification -from awips.dataaccess.PyGridNotification import PyGridNotification +from ufpy.dataaccess.PyGeometryNotification import PyGeometryNotification +from ufpy.dataaccess.PyGridNotification import PyGridNotification THRIFT_HOST = subprocess.check_output( @@ -89,7 +89,7 @@ if sys.modules.has_key('jep'): import JepRouter router = JepRouter else: - from awips.dataaccess import ThriftClientRouter + from ufpy.dataaccess import ThriftClientRouter router = ThriftClientRouter.ThriftClientRouter(THRIFT_HOST) USING_NATIVE_THRIFT = True diff --git a/pythonPackages/ufpy/dataaccess/DataQueue.py b/pythonPackages/ufpy/dataaccess/DataQueue.py index 75fd7daa3b..f6babd5b93 100644 --- a/pythonPackages/ufpy/dataaccess/DataQueue.py +++ b/pythonPackages/ufpy/dataaccess/DataQueue.py @@ -33,7 +33,7 @@ # 07/29/16 2416 tgurney Initial creation # -from awips.dataaccess import DataNotificationLayer as DNL +from ufpy.dataaccess import DataNotificationLayer as DNL import time from threading import Thread diff --git a/pythonPackages/ufpy/dataaccess/PyData.py b/pythonPackages/ufpy/dataaccess/PyData.py index 68943510f2..0d1a1effe0 100644 --- a/pythonPackages/ufpy/dataaccess/PyData.py +++ b/pythonPackages/ufpy/dataaccess/PyData.py @@ -31,7 +31,7 @@ # # -from awips.dataaccess import IData +from ufpy.dataaccess import IData class PyData(IData): diff --git a/pythonPackages/ufpy/dataaccess/PyGeometryData.py b/pythonPackages/ufpy/dataaccess/PyGeometryData.py index 99e5bb0627..d1b426a87b 100644 --- a/pythonPackages/ufpy/dataaccess/PyGeometryData.py +++ b/pythonPackages/ufpy/dataaccess/PyGeometryData.py @@ -36,8 +36,8 @@ # # -from awips.dataaccess import IGeometryData -from awips.dataaccess import PyData +from ufpy.dataaccess import IGeometryData +from ufpy.dataaccess import PyData class PyGeometryData(IGeometryData, PyData.PyData): diff --git a/pythonPackages/ufpy/dataaccess/PyGeometryNotification.py b/pythonPackages/ufpy/dataaccess/PyGeometryNotification.py index d92f9a1b25..61846e26fe 100644 --- a/pythonPackages/ufpy/dataaccess/PyGeometryNotification.py +++ b/pythonPackages/ufpy/dataaccess/PyGeometryNotification.py @@ -29,7 +29,7 @@ # 07/22/16 2416 tgurney Initial creation # -from awips.dataaccess.PyNotification import PyNotification +from ufpy.dataaccess.PyNotification import PyNotification from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint class PyGeometryNotification(PyNotification): diff --git a/pythonPackages/ufpy/dataaccess/PyGridData.py b/pythonPackages/ufpy/dataaccess/PyGridData.py index ab6c5e5600..991e1d944e 100644 --- a/pythonPackages/ufpy/dataaccess/PyGridData.py +++ b/pythonPackages/ufpy/dataaccess/PyGridData.py @@ -28,7 +28,9 @@ # Date Ticket# Engineer Description # ------------ ---------- ----------- -------------------------- # 06/03/13 #2023 dgilling Initial Creation. +# 10/13/16 #5916 bsteffen Correct grid shape, allow lat/lon # 11/10/16 #5900 bsteffen Correct grid shape +# to be requested by a delegate # # @@ -36,8 +38,8 @@ import numpy import warnings -from awips.dataaccess import IGridData -from awips.dataaccess import PyData +from ufpy.dataaccess import IGridData +from ufpy.dataaccess import PyData NO_UNIT_CONVERT_WARNING = """ The ability to unit convert grid data is not currently available in this version of the Data Access Framework. @@ -46,7 +48,7 @@ The ability to unit convert grid data is not currently available in this version class PyGridData(IGridData, PyData.PyData): - def __init__(self, gridDataRecord, nx, ny, latLonGrid): + def __init__(self, gridDataRecord, nx, ny, latLonGrid = None, latLonDelegate = None): PyData.PyData.__init__(self, gridDataRecord) nx = nx ny = ny @@ -54,6 +56,8 @@ class PyGridData(IGridData, PyData.PyData): self.__unit = gridDataRecord.getUnit() self.__gridData = numpy.reshape(numpy.array(gridDataRecord.getGridData()), (ny, nx)) self.__latLonGrid = latLonGrid + self.__latLonDelegate = latLonDelegate + def getParameter(self): return self.__parameter @@ -70,4 +74,8 @@ class PyGridData(IGridData, PyData.PyData): return self.__gridData def getLatLonCoords(self): + if self.__latLonGrid is not None: + return self.__latLonGrid + elif self.__latLonDelegate is not None: + return self.__latLonDelegate() return self.__latLonGrid diff --git a/pythonPackages/ufpy/dataaccess/PyGridNotification.py b/pythonPackages/ufpy/dataaccess/PyGridNotification.py index b726020ea9..ba19d2380f 100644 --- a/pythonPackages/ufpy/dataaccess/PyGridNotification.py +++ b/pythonPackages/ufpy/dataaccess/PyGridNotification.py @@ -29,7 +29,7 @@ # 06/03/16 2416 rjpeter Initial Creation. # -from awips.dataaccess.PyNotification import PyNotification +from ufpy.dataaccess.PyNotification import PyNotification from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint class PyGridNotification(PyNotification): diff --git a/pythonPackages/ufpy/dataaccess/PyNotification.py b/pythonPackages/ufpy/dataaccess/PyNotification.py index 79c25c52c0..0f7e8832cc 100644 --- a/pythonPackages/ufpy/dataaccess/PyNotification.py +++ b/pythonPackages/ufpy/dataaccess/PyNotification.py @@ -37,10 +37,10 @@ import time import traceback import dynamicserialize -from awips.dataaccess import DataAccessLayer -from awips.dataaccess import INotificationSubscriber -from awips.QpidSubscriber import QpidSubscriber -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer +from ufpy.dataaccess import INotificationSubscriber +from ufpy.QpidSubscriber import QpidSubscriber +from ufpy.ThriftClient import ThriftRequestException from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime diff --git a/pythonPackages/ufpy/dataaccess/SoundingsSupport.py b/pythonPackages/ufpy/dataaccess/SoundingsSupport.py index b4d5fc0e70..2f0bb2043e 100644 --- a/pythonPackages/ufpy/dataaccess/SoundingsSupport.py +++ b/pythonPackages/ufpy/dataaccess/SoundingsSupport.py @@ -34,8 +34,8 @@ from collections import defaultdict from shapely.geometry import Point -from awips import DateTimeConverter -from awips.dataaccess import DataAccessLayer +from ufpy import DateTimeConverter +from ufpy.dataaccess import DataAccessLayer from dynamicserialize.dstypes.com.raytheon.uf.common.time import DataTime from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.level import Level diff --git a/pythonPackages/ufpy/dataaccess/ThriftClientRouter.py b/pythonPackages/ufpy/dataaccess/ThriftClientRouter.py index 182d0eb6e3..2fb4565c60 100644 --- a/pythonPackages/ufpy/dataaccess/ThriftClientRouter.py +++ b/pythonPackages/ufpy/dataaccess/ThriftClientRouter.py @@ -39,7 +39,8 @@ # getRequiredIdentifiers() and # getOptionalIdentifiers() # 08/01/16 2416 tgurney Add getNotificationFilter() -# 11/10/16 5900 bsteffen Correct grid shape +# 10/13/16 5916 bsteffen Correct grid shape, allow lazy grid lat/lon +# 10/26/16 5919 njensen Speed up geometry creation in getGeometryData() # @@ -51,6 +52,7 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableTimesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGeometryDataRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridDataRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetGridLatLonRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableParametersRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetAvailableLevelsRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetRequiredIdentifiersRequest @@ -59,15 +61,44 @@ from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import G from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetSupportedDatatypesRequest from dynamicserialize.dstypes.com.raytheon.uf.common.dataaccess.request import GetNotificationFilterRequest -from awips import ThriftClient -from awips.dataaccess import PyGeometryData -from awips.dataaccess import PyGridData +from ufpy import ThriftClient +from ufpy.dataaccess import PyGeometryData +from ufpy.dataaccess import PyGridData + + +class LazyGridLatLon(object): + + def __init__(self, client, nx, ny, envelope, crsWkt): + self._latLonGrid = None + self._client = client + self._request = GetGridLatLonRequest() + self._request.setNx(nx) + self._request.setNy(ny) + self._request.setEnvelope(envelope) + self._request.setCrsWkt(crsWkt) + + def __call__(self): + # Its important that the data is cached internally so that if multiple + # GridData are sharing the same delegate then they can also share a + # single request for the LatLon information. + if self._latLonGrid is None: + response = self._client.sendRequest(self._request) + nx = response.getNx() + ny = response.getNy() + latData = numpy.reshape(numpy.array(response.getLats()), (ny, nx)) + lonData = numpy.reshape(numpy.array(response.getLons()), (ny, nx)) + self._latLonGrid = (lonData, latData) + return self._latLonGrid class ThriftClientRouter(object): def __init__(self, host='localhost'): self._client = ThriftClient.ThriftClient(host) + self._lazyLoadGridLatLon = False + + def setLazyLoadGridLatLon(self, lazyLoadGridLatLon): + self._lazyLoadGridLatLon = lazyLoadGridLatLon def getAvailableTimes(self, request, refTimeOnly): timesRequest = GetAvailableTimesRequest() @@ -78,6 +109,7 @@ class ThriftClientRouter(object): def getGridData(self, request, times): gridDataRequest = GetGridDataRequest() + gridDataRequest.setIncludeLatLonData(not self._lazyLoadGridLatLon) gridDataRequest.setRequestParameters(request) # if we have an iterable times instance, then the user must have asked # for grid data with the List of DataTime objects @@ -95,15 +127,28 @@ class ThriftClientRouter(object): for location in locNames: nx = response.getSiteNxValues()[location] ny = response.getSiteNyValues()[location] - latData = numpy.reshape(numpy.array(response.getSiteLatGrids()[location]), (ny, nx)) - lonData = numpy.reshape(numpy.array(response.getSiteLonGrids()[location]), (ny, nx)) - locSpecificData[location] = (nx, ny, (lonData, latData)) - + if self._lazyLoadGridLatLon: + envelope = response.getSiteEnvelopes()[location] + crsWkt = response.getSiteCrsWkt()[location] + delegate = LazyGridLatLon( + self._client, nx, ny, envelope, crsWkt) + locSpecificData[location] = (nx, ny, delegate) + else: + latData = numpy.reshape(numpy.array( + response.getSiteLatGrids()[location]), (ny, nx)) + lonData = numpy.reshape(numpy.array( + response.getSiteLonGrids()[location]), (ny, nx)) + locSpecificData[location] = (nx, ny, (lonData, latData)) retVal = [] for gridDataRecord in response.getGridData(): locationName = gridDataRecord.getLocationName() locData = locSpecificData[locationName] - retVal.append(PyGridData.PyGridData(gridDataRecord, locData[0], locData[1], locData[2])) + if self._lazyLoadGridLatLon: + retVal.append(PyGridData.PyGridData(gridDataRecord, locData[ + 0], locData[1], latLonDelegate=locData[2])) + else: + retVal.append(PyGridData.PyGridData( + gridDataRecord, locData[0], locData[1], locData[2])) return retVal def getGeometryData(self, request, times): @@ -121,10 +166,9 @@ class ThriftClientRouter(object): response = self._client.sendRequest(geoDataRequest) geometries = [] for wkb in response.getGeometryWKBs(): - # convert the wkb to a bytearray with only positive values - byteArrWKB = bytearray(map(lambda x: x % 256,wkb.tolist())) - # convert the bytearray to a byte string and load it. - geometries.append(shapely.wkb.loads(str(byteArrWKB))) + # the wkb is a numpy.ndarray of dtype int8 + # convert the bytearray to a byte string and load it + geometries.append(shapely.wkb.loads(wkb.tostring())) retVal = [] for geoDataRecord in response.getGeoData(): @@ -175,7 +219,7 @@ class ThriftClientRouter(object): response = self._client.sendRequest(idValReq) return response - def newDataRequest(self, datatype, parameters=[], levels=[], locationNames = [], envelope=None, **kwargs): + def newDataRequest(self, datatype, parameters=[], levels=[], locationNames=[], envelope=None, **kwargs): req = DefaultDataRequest() if datatype: req.setDatatype(datatype) @@ -195,9 +239,9 @@ class ThriftClientRouter(object): def getSupportedDatatypes(self): response = self._client.sendRequest(GetSupportedDatatypesRequest()) return response - + def getNotificationFilter(self, request): notifReq = GetNotificationFilterRequest() notifReq.setRequestParameters(request) response = self._client.sendRequest(notifReq) - return response \ No newline at end of file + return response diff --git a/pythonPackages/ufpy/dataaccess/__init__.py b/pythonPackages/ufpy/dataaccess/__init__.py index e21257270f..898f7949ad 100644 --- a/pythonPackages/ufpy/dataaccess/__init__.py +++ b/pythonPackages/ufpy/dataaccess/__init__.py @@ -20,7 +20,7 @@ # -# __init__.py for awips.dataaccess package +# __init__.py for ufpy.dataaccess package # # # SOFTWARE HISTORY diff --git a/pythonPackages/ufpy/gfe/IFPClient.py b/pythonPackages/ufpy/gfe/IFPClient.py index d1e3f46936..1288094df6 100644 --- a/pythonPackages/ufpy/gfe/IFPClient.py +++ b/pythonPackages/ufpy/gfe/IFPClient.py @@ -18,7 +18,7 @@ # further licensing information. ## -from awips import ThriftClient +from ufpy import ThriftClient from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import DatabaseID from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.db.objects import ParmID diff --git a/pythonPackages/ufpy/gfe/__init__.py b/pythonPackages/ufpy/gfe/__init__.py index 0e5965abfb..f2a16d5c47 100644 --- a/pythonPackages/ufpy/gfe/__init__.py +++ b/pythonPackages/ufpy/gfe/__init__.py @@ -20,7 +20,7 @@ # -# __init__.py for awips.gfe package +# __init__.py for ufpy.gfe package # # # SOFTWARE HISTORY diff --git a/pythonPackages/ufpy/qpidingest.py b/pythonPackages/ufpy/qpidingest.py index 7a920a5e33..8a7b2404ad 100644 --- a/pythonPackages/ufpy/qpidingest.py +++ b/pythonPackages/ufpy/qpidingest.py @@ -59,9 +59,13 @@ # .... # 06/13/2013 DR 16242 D. Friedman Add Qpid authentication info # 03/06/2014 DR 17907 D. Friedman Workaround for issue QPID-5569 +# 02/16/2017 DR 6084 bsteffen Support ssl connections # #=============================================================================== +import os +import os.path + import qpid from qpid.util import connect from qpid.connection import Connection @@ -71,17 +75,31 @@ QPID_USERNAME = 'guest' QPID_PASSWORD = 'guest' class IngestViaQPID: - def __init__(self, host='localhost', port=5672): + def __init__(self, host='localhost', port=5672, ssl=None): ''' Connect to QPID and make bindings to route message to external.dropbox queue @param host: string hostname of computer running EDEX and QPID (default localhost) @param port: integer port used to connect to QPID (default 5672) + @param ssl: boolean to determine whether ssl is used, default value of None will use ssl only if a client certificate is found. ''' try: # - self.socket = connect(host, port) - self.connection = Connection (sock=self.socket, username=QPID_USERNAME, password=QPID_PASSWORD) + socket = connect(host, port) + if "QPID_SSL_CERT_DB" in os.environ: + certdb = os.environ["QPID_SSL_CERT_DB"] + else: + certdb = os.path.expanduser("~/.qpid/") + if "QPID_SSL_CERT_NAME" in os.environ: + certname = os.environ["QPID_SSL_CERT_NAME"] + else: + certname = QPID_USERNAME + certfile = os.path.join(certdb, certname + ".crt") + if ssl or (ssl is None and os.path.exists(certfile)): + keyfile = os.path.join(certdb, certname + ".key") + trustfile = os.path.join(certdb, "root.crt") + socket = qpid.util.ssl(socket, keyfile=keyfile, certfile=certfile, ca_certs=trustfile) + self.connection = Connection (sock=socket, username=QPID_USERNAME, password=QPID_PASSWORD) self.connection.start() self.session = self.connection.session(str(uuid4())) self.session.exchange_bind(exchange='amq.direct', queue='external.dropbox', binding_key='external.dropbox') @@ -108,4 +126,4 @@ class IngestViaQPID: there are no threads left open ''' self.session.close(timeout=10) - print 'Connection to Qpid closed' \ No newline at end of file + print 'Connection to Qpid closed' diff --git a/pythonPackages/ufpy/test/Test b/pythonPackages/ufpy/test/Test index 00d71d449b..4ff1a0ba84 100644 --- a/pythonPackages/ufpy/test/Test +++ b/pythonPackages/ufpy/test/Test @@ -39,7 +39,7 @@ import os import logging -from awips import AlertVizHandler +from ufpy import AlertVizHandler import Record avh = AlertVizHandler.AlertVizHandler(host=os.getenv("BROKER_ADDR","localhost"), port=9581, category='LOCAL', source='ANNOUNCER', level=logging.NOTSET) diff --git a/pythonPackages/ufpy/test/__init__.py b/pythonPackages/ufpy/test/__init__.py index 65dffe03c5..ae5228cd62 100644 --- a/pythonPackages/ufpy/test/__init__.py +++ b/pythonPackages/ufpy/test/__init__.py @@ -20,7 +20,7 @@ # -# __init__.py for awips package +# __init__.py for ufpy package # # # SOFTWARE HISTORY diff --git a/pythonPackages/ufpy/test/dafTests/__init__.py b/pythonPackages/ufpy/test/dafTests/__init__.py index 6041dc1ad1..5ce6ada4e0 100644 --- a/pythonPackages/ufpy/test/dafTests/__init__.py +++ b/pythonPackages/ufpy/test/dafTests/__init__.py @@ -20,7 +20,7 @@ # -# __init__.py for awips.test.dafTests package +# __init__.py for ufpy.test.dafTests package # # # SOFTWARE HISTORY diff --git a/pythonPackages/ufpy/test/dafTests/baseBufrMosTestCase.py b/pythonPackages/ufpy/test/dafTests/baseBufrMosTestCase.py index 652d838963..a9007a449f 100644 --- a/pythonPackages/ufpy/test/dafTests/baseBufrMosTestCase.py +++ b/pythonPackages/ufpy/test/dafTests/baseBufrMosTestCase.py @@ -18,9 +18,12 @@ # further licensing information. ## -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL +from shapely.geometry import box import baseDafTestCase +import params +import unittest # # Base TestCase for BufrMos* tests. @@ -31,14 +34,17 @@ import baseDafTestCase # ------------ ---------- ----------- -------------------------- # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup -# +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 5981 tgurney Add envelope test # # class BufrMosTestCase(baseDafTestCase.DafTestCase): """Base class for testing DAF support of bufrmos data""" - + + data_params = "temperature", "dewpoint" + def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) self.runParametersTest(req) @@ -49,11 +55,19 @@ class BufrMosTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("temperature", "dewpoint") + req.setLocationNames(params.OBS_STATION) + req.setParameters(*self.data_params) self.runGeometryDataTest(req) + + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters(*self.data_params) + req.setEnvelope(params.ENVELOPE) + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) diff --git a/pythonPackages/ufpy/test/dafTests/baseDafTestCase.py b/pythonPackages/ufpy/test/dafTests/baseDafTestCase.py index 9275082083..8a7225ca4f 100644 --- a/pythonPackages/ufpy/test/dafTests/baseDafTestCase.py +++ b/pythonPackages/ufpy/test/dafTests/baseDafTestCase.py @@ -20,8 +20,8 @@ from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException import os import unittest @@ -50,6 +50,8 @@ import unittest # 10/05/16 5926 dgilling Better checks in runGeometryDataTest. # 11/08/16 5985 tgurney Do not check data times on # time-agnostic data +# 03/13/17 5981 tgurney Do not check valid period on +# data time # # @@ -166,10 +168,13 @@ class DafTestCase(unittest.TestCase): self.assertIsNotNone(geomData) if times: self.assertNotEqual(len(geomData), 0) + if not geomData: + raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: - if checkDataTimes and times: + if (checkDataTimes and times and + "PERIOD_USED" not in record.getDataTime().getUtilityFlags()): self.assertIn(record.getDataTime(), times[:self.numTimesToLimit]) print("geometry=" + str(record.getGeometry()), end="") for p in req.getParameters(): @@ -184,6 +189,8 @@ class DafTestCase(unittest.TestCase): """ geomData = DAL.getGeometryData(req, timeRange) self.assertIsNotNone(geomData) + if not geomData: + raise unittest.SkipTest("No data available") print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") for record in geomData[:self.sampleDataLimit]: @@ -207,6 +214,8 @@ class DafTestCase(unittest.TestCase): times = DafTestCase.getTimesIfSupported(req) gridData = DAL.getGridData(req, times[:self.numTimesToLimit]) self.assertIsNotNone(gridData) + if not gridData: + raise unittest.SkipTest("No data available") print("Number of grid records: " + str(len(gridData))) if len(gridData) > 0: print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") diff --git a/pythonPackages/ufpy/test/dafTests/baseRadarTestCase.py b/pythonPackages/ufpy/test/dafTests/baseRadarTestCase.py new file mode 100644 index 0000000000..085fea0203 --- /dev/null +++ b/pythonPackages/ufpy/test/dafTests/baseRadarTestCase.py @@ -0,0 +1,194 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from __future__ import print_function +from shapely.geometry import box +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException + +import baseDafTestCase +import params +import unittest + +# +# Tests common to all radar factories +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 01/19/16 4795 mapeters Initial Creation. +# 04/11/16 5548 tgurney Cleanup +# 04/18/16 5548 tgurney More cleanup +# 04/26/16 5587 tgurney Move identifier values tests +# out of base class +# 06/01/16 5587 tgurney Update testGetIdentifierValues +# 06/08/16 5574 mapeters Add advanced query tests +# 06/13/16 5574 tgurney Fix checks for None +# 06/14/16 5548 tgurney Undo previous change (broke +# test) +# 06/30/16 5725 tgurney Add test for NOT IN +# 08/25/16 2671 tgurney Rename to baseRadarTestCase +# and move factory-specific +# tests +# 12/07/16 5981 tgurney Parameterize +# +# + + +class BaseRadarTestCase(baseDafTestCase.DafTestCase): + """Tests common to all radar factories""" + + # datatype is specified by subclass + datatype = None + + radarLoc = params.RADAR.lower() + + def testGetAvailableParameters(self): + req = DAL.newDataRequest(self.datatype) + self.runParametersTest(req) + + def testGetAvailableLocations(self): + req = DAL.newDataRequest(self.datatype) + self.runLocationsTest(req) + + def testGetAvailableLevels(self): + req = DAL.newDataRequest(self.datatype) + self.runLevelsTest(req) + + def testGetAvailableLevelsWithInvalidLevelIdentifierThrowsException(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('level.one.field', 'invalidLevelField') + with self.assertRaises(ThriftRequestException) as cm: + self.runLevelsTest(req) + self.assertIn('IncompatibleRequestException', str(cm.exception)) + + def testGetAvailableTimes(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + self.runTimesTest(req) + + def testGetIdentifierValues(self): + req = DAL.newDataRequest(self.datatype) + optionalIds = set(DAL.getOptionalIdentifiers(req)) + requiredIds = set(DAL.getRequiredIdentifiers(req)) + self.runGetIdValuesTest(optionalIds | requiredIds) + + def testGetInvalidIdentifierValuesThrowsException(self): + self.runInvalidIdValuesTest() + + def testGetNonexistentIdentifierValuesThrowsException(self): + self.runNonexistentIdValuesTest() + + def runConstraintTest(self, key, operator, value): + raise NotImplementedError + + def testGetDataWithEqualsString(self): + gridData = self.runConstraintTest('icao', '=', self.radarLoc) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithEqualsUnicode(self): + gridData = self.runConstraintTest('icao', '=', unicode(self.radarLoc)) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithEqualsInt(self): + gridData = self.runConstraintTest('icao', '=', 1000) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), 1000) + + def testGetDataWithEqualsLong(self): + gridData = self.runConstraintTest('icao', '=', 1000L) + for record in gridData: + self.assertEqual(record.getAttribute('icao'), 1000) + + def testGetDataWithEqualsFloat(self): + gridData = self.runConstraintTest('icao', '=', 1.0) + for record in gridData: + self.assertEqual(round(record.getAttribute('icao'), 1), 1.0) + + def testGetDataWithEqualsNone(self): + gridData = self.runConstraintTest('icao', '=', None) + for record in gridData: + self.assertIsNone(record.getAttribute('icao')) + + def testGetDataWithNotEquals(self): + gridData = self.runConstraintTest('icao', '!=', self.radarLoc) + for record in gridData: + self.assertNotEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithNotEqualsNone(self): + gridData = self.runConstraintTest('icao', '!=', None) + for record in gridData: + self.assertIsNotNone(record.getAttribute('icao')) + + def testGetDataWithGreaterThan(self): + gridData = self.runConstraintTest('icao', '>', self.radarLoc) + for record in gridData: + self.assertGreater(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithLessThan(self): + gridData = self.runConstraintTest('icao', '<', self.radarLoc) + for record in gridData: + self.assertLess(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithGreaterThanEquals(self): + gridData = self.runConstraintTest('icao', '>=', self.radarLoc) + for record in gridData: + self.assertGreaterEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithLessThanEquals(self): + gridData = self.runConstraintTest('icao', '<=', self.radarLoc) + for record in gridData: + self.assertLessEqual(record.getAttribute('icao'), self.radarLoc) + + def testGetDataWithInTuple(self): + gridData = self.runConstraintTest('icao', 'in', (self.radarLoc, 'tpbi')) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithInList(self): + gridData = self.runConstraintTest('icao', 'in', [self.radarLoc, 'tpbi']) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithInGenerator(self): + generator = (item for item in (self.radarLoc, 'tpbi')) + gridData = self.runConstraintTest('icao', 'in', generator) + for record in gridData: + self.assertIn(record.getAttribute('icao'), (self.radarLoc, 'tpbi')) + + def testGetDataWithNotInList(self): + gridData = self.runConstraintTest('icao', 'not in', ['zzzz', self.radarLoc]) + for record in gridData: + self.assertNotIn(record.getAttribute('icao'), ('zzzz', self.radarLoc)) + + def testGetDataWithInvalidConstraintTypeThrowsException(self): + with self.assertRaises(ValueError): + self.runConstraintTest('icao', 'junk', self.radarLoc) + + def testGetDataWithInvalidConstraintValueThrowsException(self): + with self.assertRaises(TypeError): + self.runConstraintTest('icao', '=', {}) + + def testGetDataWithEmptyInConstraintThrowsException(self): + with self.assertRaises(ValueError): + self.runConstraintTest('icao', 'in', []) diff --git a/pythonPackages/ufpy/test/dafTests/params.py b/pythonPackages/ufpy/test/dafTests/params.py new file mode 100644 index 0000000000..bdc6da7ed6 --- /dev/null +++ b/pythonPackages/ufpy/test/dafTests/params.py @@ -0,0 +1,43 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + + +# +# Site-specific parameters for DAF tests +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 12/07/16 5981 tgurney Initial creation +# 12/15/16 5981 tgurney Add ENVELOPE +# +# + +from shapely.geometry import box + +AIRPORT = 'OMA' +OBS_STATION = 'KOMA' +SITE_ID = 'OAX' +STATION_ID = '72558' +RADAR = 'KOAX' +SAMPLE_AREA = (-97.0, 41.0, -96.0, 42.0) + +ENVELOPE = box(*SAMPLE_AREA) \ No newline at end of file diff --git a/pythonPackages/ufpy/test/dafTests/testAcars.py b/pythonPackages/ufpy/test/dafTests/testAcars.py index 9ab9167e88..b43cb3f35d 100644 --- a/pythonPackages/ufpy/test/dafTests/testAcars.py +++ b/pythonPackages/ufpy/test/dafTests/testAcars.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase import unittest diff --git a/pythonPackages/ufpy/test/dafTests/testAirep.py b/pythonPackages/ufpy/test/dafTests/testAirep.py index 9d9fc96524..a4fc5fd98f 100644 --- a/pythonPackages/ufpy/test/dafTests/testAirep.py +++ b/pythonPackages/ufpy/test/dafTests/testAirep.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase diff --git a/pythonPackages/ufpy/test/dafTests/testBinLightning.py b/pythonPackages/ufpy/test/dafTests/testBinLightning.py index da273eb09e..468a6b13be 100644 --- a/pythonPackages/ufpy/test/dafTests/testBinLightning.py +++ b/pythonPackages/ufpy/test/dafTests/testBinLightning.py @@ -19,8 +19,8 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint diff --git a/pythonPackages/ufpy/test/dafTests/testBufrMosHpc.py b/pythonPackages/ufpy/test/dafTests/testBufrMosHpc.py index e1ab295a67..ab1b338b8f 100644 --- a/pythonPackages/ufpy/test/dafTests/testBufrMosHpc.py +++ b/pythonPackages/ufpy/test/dafTests/testBufrMosHpc.py @@ -19,9 +19,10 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseBufrMosTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Inherit all tests # # @@ -42,11 +45,6 @@ class BufrMosHpcTestCase(baseBufrMosTestCase.BufrMosTestCase): """Test DAF support for bufrmosHPC data""" datatype = "bufrmosHPC" + data_params = "forecastHr", "maxTemp24Hour" - # Most tests inherited from superclass - - def testGetGeometryData(self): - req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("forecastHr", "maxTemp24Hour") - self.runGeometryDataTest(req) + # All tests inherited from superclass \ No newline at end of file diff --git a/pythonPackages/ufpy/test/dafTests/testBufrMosMrf.py b/pythonPackages/ufpy/test/dafTests/testBufrMosMrf.py index 27d1b0f827..6ce3d102b7 100644 --- a/pythonPackages/ufpy/test/dafTests/testBufrMosMrf.py +++ b/pythonPackages/ufpy/test/dafTests/testBufrMosMrf.py @@ -19,9 +19,10 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseBufrMosTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Inherit all tests # # @@ -42,11 +45,6 @@ class BufrMosMrfTestCase(baseBufrMosTestCase.BufrMosTestCase): """Test DAF support for bufrmosMRF data""" datatype = "bufrmosMRF" + data_params = "forecastHr", "maxTempDay" - # Most tests inherited from superclass - - def testGetGeometryData(self): - req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") - req.setParameters("forecastHr", "maxTempDay") - self.runGeometryDataTest(req) + # All tests inherited from superclass diff --git a/pythonPackages/ufpy/test/dafTests/testBufrUa.py b/pythonPackages/ufpy/test/dafTests/testBufrUa.py index 42bfc0518a..8263a2ba37 100644 --- a/pythonPackages/ufpy/test/dafTests/testBufrUa.py +++ b/pythonPackages/ufpy/test/dafTests/testBufrUa.py @@ -19,10 +19,11 @@ # # from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -38,6 +39,8 @@ import unittest # 06/09/16 5587 bsteffen Add getIdentifierValues tests # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 5981 tgurney Add envelope test # # @@ -47,8 +50,7 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): datatype = "bufrua" - location = "72558" - """stationid corresponding to KOAX""" + location = params.STATION_ID def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) @@ -91,6 +93,14 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): print("getGeometryData() complete\n\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters("staName", "rptType") + req.setEnvelope(params.ENVELOPE) + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) + def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) optionalIds = set(DAL.getOptionalIdentifiers(req)) @@ -133,7 +143,6 @@ class BufrUaTestCase(baseDafTestCase.DafTestCase): # No float test because no float identifiers are available - def testGetDataWithEqualsNone(self): geometryData = self._runConstraintTest('reportType', '=', None) for record in geometryData: diff --git a/pythonPackages/ufpy/test/dafTests/testClimate.py b/pythonPackages/ufpy/test/dafTests/testClimate.py index 6bb98ab7f6..40381aa1bd 100644 --- a/pythonPackages/ufpy/test/dafTests/testClimate.py +++ b/pythonPackages/ufpy/test/dafTests/testClimate.py @@ -22,10 +22,11 @@ from __future__ import print_function import datetime from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException import baseDafTestCase +import params import unittest # @@ -44,6 +45,8 @@ import unittest # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN # 10/06/16 5926 dgilling Add additional time and location tests. +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test # # @@ -52,6 +55,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """Test DAF support for climate data""" datatype = 'climate' + obsStation = params.OBS_STATION def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) @@ -104,7 +108,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_monthly') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('maxtemp_mon', 'min_sea_press') self.runTimesTest(req) @@ -115,7 +119,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_asos_daily') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('maxtemp_cal', 'min_press') self.runTimesTest(req) @@ -126,7 +130,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.cli_mon_season_yr') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('max_temp', 'precip_total') self.runTimesTest(req) @@ -137,7 +141,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): """ req = DAL.newDataRequest(self.datatype) req.addIdentifier('table', 'public.daily_climate') - req.setLocationNames('KOMA', 'KABR', 'KDMO') + req.setLocationNames(self.obsStation, 'KABR', 'KDMO') req.setParameters('max_temp', 'precip', 'avg_wind_speed') self.runTimesTest(req) @@ -155,6 +159,15 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): req.setParameters('maxtemp_mon', 'min_sea_press') self.runGeometryDataTest(req) + def testGetGeometryDataWithEnvelopeThrowsException(self): + # Envelope is not used + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('table', 'public.cli_asos_monthly') + req.setParameters('maxtemp_mon', 'min_sea_press') + req.setEnvelope(params.ENVELOPE) + with self.assertRaises(Exception): + data = self.runGeometryDataTest(req) + def testGetGeometryDataForYearAndDayOfYearTable(self): """ Test retrieval of data for a climo table that uses year and @@ -243,14 +256,14 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): return self.runGeometryDataTest(req) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('station_code', '=', 'KOMA') + geometryData = self._runConstraintTest('station_code', '=', self.obsStation) for record in geometryData: - self.assertEqual(record.getString('station_code'), 'KOMA') + self.assertEqual(record.getString('station_code'), self.obsStation) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('station_code', '=', u'KOMA') + geometryData = self._runConstraintTest('station_code', '=', unicode(self.obsStation)) for record in geometryData: - self.assertEqual(record.getString('station_code'), 'KOMA') + self.assertEqual(record.getString('station_code'), self.obsStation) def testGetDataWithEqualsInt(self): geometryData = self._runConstraintTest('avg_daily_max', '=', 70) @@ -272,9 +285,9 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): self.assertEqual(len(geometryData), 0) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('station_code', '!=', 'KOMA') + geometryData = self._runConstraintTest('station_code', '!=', self.obsStation) for record in geometryData: - self.assertNotEqual(record.getString('station_code'), 'KOMA') + self.assertNotEqual(record.getString('station_code'), self.obsStation) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('station_code', '!=', None) @@ -302,19 +315,19 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): self.assertLessEqual(record.getNumber('avg_daily_max'), 70) def testGetDataWithInTuple(self): - collection = ('KOMA', 'KABR') + collection = (self.obsStation, 'KABR') geometryData = self._runConstraintTest('station_code', 'in', collection) for record in geometryData: self.assertIn(record.getString('station_code'), collection) def testGetDataWithInList(self): - collection = ['KOMA', 'KABR'] + collection = [self.obsStation, 'KABR'] geometryData = self._runConstraintTest('station_code', 'in', collection) for record in geometryData: self.assertIn(record.getString('station_code'), collection) def testGetDataWithInGenerator(self): - collection = ('KOMA', 'KABR') + collection = (self.obsStation, 'KABR') generator = (item for item in collection) geometryData = self._runConstraintTest('station_code', 'in', generator) for record in geometryData: @@ -328,7 +341,7 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('station_code', 'junk', 'KOMA') + self._runConstraintTest('station_code', 'junk', self.obsStation) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): @@ -417,4 +430,3 @@ class ClimateTestCase(baseDafTestCase.DafTestCase): endTime = datetime.datetime(2009, 3, 31) tr = TimeRange(startTime, endTime) self.runGeometryDataTestWithTimeRange(req, tr) - diff --git a/pythonPackages/ufpy/test/dafTests/testCombinedTimeQuery.py b/pythonPackages/ufpy/test/dafTests/testCombinedTimeQuery.py index 429edc3124..92a9750c2e 100644 --- a/pythonPackages/ufpy/test/dafTests/testCombinedTimeQuery.py +++ b/pythonPackages/ufpy/test/dafTests/testCombinedTimeQuery.py @@ -18,9 +18,9 @@ # further licensing information. ## -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL -from awips.dataaccess import CombinedTimeQuery as CTQ +from ufpy.dataaccess import CombinedTimeQuery as CTQ import unittest import os @@ -49,7 +49,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase): def testSuccessfulQuery(self): req = DAL.newDataRequest('grid') - req.setLocationNames('RAP13') + req.setLocationNames('RUC130') req.setParameters('T','GH') req.setLevels('300MB', '500MB','700MB') times = CTQ.getAvailableTimes(req); @@ -60,7 +60,7 @@ class CombinedTimeQueryTestCase(unittest.TestCase): Test that when a parameter is only available on one of the levels that no times are returned. """ req = DAL.newDataRequest('grid') - req.setLocationNames('RAP13') + req.setLocationNames('RUC130') req.setParameters('T','GH', 'LgSP1hr') req.setLevels('300MB', '500MB','700MB','0.0SFC') times = CTQ.getAvailableTimes(req); diff --git a/pythonPackages/ufpy/test/dafTests/testCommonObsSpatial.py b/pythonPackages/ufpy/test/dafTests/testCommonObsSpatial.py index 8a3e18dd4e..984c57fd27 100644 --- a/pythonPackages/ufpy/test/dafTests/testCommonObsSpatial.py +++ b/pythonPackages/ufpy/test/dafTests/testCommonObsSpatial.py @@ -20,10 +20,11 @@ from __future__ import print_function from shapely.geometry import box -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -42,6 +43,8 @@ import unittest # 06/13/16 5574 tgurney Add advanced query tests # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Do not check data times # @@ -50,9 +53,6 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): datatype = "common_obs_spatial" - envelope = box(-97.0, 41.0, -96.0, 42.0) - """Default request area (box around KOAX)""" - def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) self.runParametersTest(req) @@ -65,19 +65,11 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): def testGetIdentifierValues(self): self.runGetIdValuesTest(['country']) - @unittest.skip('avoid EDEX error') - def testGetInvalidIdentifierValuesThrowsException(self): - self.runInvalidIdValuesTest() - - @unittest.skip('avoid EDEX error') - def testGetNonexistentIdentifierValuesThrowsException(self): - self.runNonexistentIdValuesTest() - def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) req.setParameters("name", "stationid") - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -88,7 +80,7 @@ class CommonObsSpatialTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('catalogtype', 'elevation', 'state') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('state', '=', 'NE') diff --git a/pythonPackages/ufpy/test/dafTests/testFfmp.py b/pythonPackages/ufpy/test/dafTests/testFfmp.py index 5fa7d93919..5edc20c897 100644 --- a/pythonPackages/ufpy/test/dafTests/testFfmp.py +++ b/pythonPackages/ufpy/test/dafTests/testFfmp.py @@ -20,9 +20,10 @@ from __future__ import print_function from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase +import params import unittest # @@ -46,6 +47,8 @@ import unittest # PRTM parameter since it isn't # configured for ec-oma # 11/08/16 5985 tgurney Do not check data times +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Do not check data times # # @@ -54,10 +57,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): """Test DAF support for ffmp data""" datatype = 'ffmp' + location = params.RADAR.lower() @staticmethod def addIdentifiers(req): - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('siteKey', 'hpe') req.addIdentifier('dataKey', 'hpe') req.addIdentifier('huc', 'ALL') @@ -99,8 +103,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) if id == 'accumHrs': req.setParameters('ARI6H2YR') - req.addIdentifier('wfo', 'OAX') - req.addIdentifier('siteKey', 'koax') + req.addIdentifier('wfo', params.SITE_ID) + req.addIdentifier('siteKey', self.location) req.addIdentifier('huc', 'ALL') idValues = DAL.getIdentifierValues(req, id) self.assertTrue(hasattr(idValues, '__iter__')) @@ -116,20 +120,20 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('huc', 'ALL') req.setParameters('QPFSCAN') return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('siteKey', '=', 'koax') + geometryData = self._runConstraintTest('siteKey', '=', self.location) for record in geometryData: - self.assertEqual(record.getAttribute('siteKey'), 'koax') + self.assertEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('siteKey', '=', u'koax') + geometryData = self._runConstraintTest('siteKey', '=', unicode(self.location)) for record in geometryData: - self.assertEqual(record.getAttribute('siteKey'), 'koax') + self.assertEqual(record.getAttribute('siteKey'), self.location) # No numeric tests since no numeric identifiers are available that support # RequestConstraints. @@ -140,9 +144,9 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self.assertIsNone(record.getAttribute('siteKey')) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('siteKey', '!=', 'koax') + geometryData = self._runConstraintTest('siteKey', '!=', self.location) for record in geometryData: - self.assertNotEqual(record.getAttribute('siteKey'), 'koax') + self.assertNotEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('siteKey', '!=', None) @@ -150,40 +154,40 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self.assertIsNotNone(record.getAttribute('siteKey')) def testGetDataWithGreaterThan(self): - geometryData = self._runConstraintTest('siteKey', '>', 'koax') + geometryData = self._runConstraintTest('siteKey', '>', self.location) for record in geometryData: - self.assertGreater(record.getAttribute('siteKey'), 'koax') + self.assertGreater(record.getAttribute('siteKey'), self.location) def testGetDataWithLessThan(self): - geometryData = self._runConstraintTest('siteKey', '<', 'koax') + geometryData = self._runConstraintTest('siteKey', '<', self.location) for record in geometryData: - self.assertLess(record.getAttribute('siteKey'), 'koax') + self.assertLess(record.getAttribute('siteKey'), self.location) def testGetDataWithGreaterThanEquals(self): - geometryData = self._runConstraintTest('siteKey', '>=', 'koax') + geometryData = self._runConstraintTest('siteKey', '>=', self.location) for record in geometryData: - self.assertGreaterEqual(record.getAttribute('siteKey'), 'koax') + self.assertGreaterEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithLessThanEquals(self): - geometryData = self._runConstraintTest('siteKey', '<=', 'koax') + geometryData = self._runConstraintTest('siteKey', '<=', self.location) for record in geometryData: - self.assertLessEqual(record.getAttribute('siteKey'), 'koax') + self.assertLessEqual(record.getAttribute('siteKey'), self.location) def testGetDataWithInList(self): - collection = ['koax', 'kuex'] + collection = [self.location, 'kuex'] geometryData = self._runConstraintTest('siteKey', 'in', collection) for record in geometryData: self.assertIn(record.getAttribute('siteKey'), collection) def testGetDataWithNotInList(self): - collection = ['koax', 'kuex'] + collection = [self.location, 'kuex'] geometryData = self._runConstraintTest('siteKey', 'not in', collection) for record in geometryData: self.assertNotIn(record.getAttribute('siteKey'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('siteKey', 'junk', 'koax') + self._runConstraintTest('siteKey', 'junk', self.location) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): @@ -194,11 +198,11 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): self._runConstraintTest('siteKey', 'in', []) def testGetDataWithSiteKeyAndDataKeyConstraints(self): - siteKeys = ['koax', 'hpe'] + siteKeys = [self.location, 'hpe'] dataKeys = ['kuex', 'kdmx'] req = DAL.newDataRequest(self.datatype) - req.addIdentifier('wfo', 'OAX') + req.addIdentifier('wfo', params.SITE_ID) req.addIdentifier('huc', 'ALL') siteKeysConstraint = RequestConstraint.new('in', siteKeys) @@ -217,8 +221,8 @@ class FfmpTestCase(baseDafTestCase.DafTestCase): def testGetGuidanceDataWithoutAccumHrsIdentifierSet(self): # Test that accumHrs identifier is not required for guidance data req = DAL.newDataRequest(self.datatype) - req.addIdentifier('wfo', 'OAX') - req.addIdentifier('siteKey', 'koax') + req.addIdentifier('wfo', params.SITE_ID) + req.addIdentifier('siteKey', self.location) req.addIdentifier('huc', 'ALL') req.setParameters('FFG0124hr') self.runGeometryDataTest(req, checkDataTimes=False) \ No newline at end of file diff --git a/pythonPackages/ufpy/test/dafTests/testGfe.py b/pythonPackages/ufpy/test/dafTests/testGfe.py index f5bdbebd85..bdc02e43d7 100644 --- a/pythonPackages/ufpy/test/dafTests/testGfe.py +++ b/pythonPackages/ufpy/test/dafTests/testGfe.py @@ -20,9 +20,11 @@ from __future__ import print_function from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL +from shapely.geometry import box, Point import baseDafTestCase +import params import unittest # @@ -41,6 +43,9 @@ import unittest # 06/17/16 5574 mapeters Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN # 11/07/16 5991 bsteffen Improve vector tests +# 12/07/16 5981 tgurney Parameterize +# 12/15/16 6040 tgurney Add testGetGridDataWithDbType +# 12/20/16 5981 tgurney Add envelope test # # @@ -62,20 +67,49 @@ class GfeTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('siteId', params.SITE_ID) self.runTimesTest(req) def testGetGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('siteId', params.SITE_ID) req.setParameters('T') self.runGridDataTest(req) + def testGetGridDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier('modelName', 'Fcst') + req.addIdentifier('siteId', params.SITE_ID) + req.setParameters('T') + req.setEnvelope(params.ENVELOPE) + gridData = self.runGridDataTest(req) + if not gridData: + raise unittest.SkipTest('no data available') + lons, lats = gridData[0].getLatLonCoords() + lons = lons.reshape(-1) + lats = lats.reshape(-1) + + # Ensure all points are within one degree of the original box + # to allow slight margin of error for reprojection distortion. + testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1, + params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 ) + + for i in range(len(lons)): + self.assertTrue(testEnv.contains(Point(lons[i], lats[i]))) + + def testGetGridDataWithDbType(self): + req = DAL.newDataRequest('gfe') + req.addIdentifier('parmId.dbId.modelName', 'Fcst') + req.addIdentifier('parmId.dbId.dbType', 'Prac') + req.setParameters('T', 'Td') + times = DAL.getAvailableTimes(req) + self.runGridDataTest(req) + def testGetVectorGridData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('modelName', 'Fcst') - req.addIdentifier('siteId', 'OAX') + req.addIdentifier('siteId', params.SITE_ID) req.setParameters('Wind') times = DAL.getAvailableTimes(req) if not(times): @@ -114,80 +148,80 @@ class GfeTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) - req.setLocationNames('OAX') + req.setLocationNames(params.SITE_ID) req.setParameters('T') return self.runGridDataTest(req) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('modelName', '=', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '=', 'Fcst') + for record in gridData: self.assertEqual(record.getAttribute('modelName'), 'Fcst') def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('modelName', '=', u'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '=', u'Fcst') + for record in gridData: self.assertEqual(record.getAttribute('modelName'), 'Fcst') # No numeric tests since no numeric identifiers are available. def testGetDataWithEqualsNone(self): - geometryData = self._runConstraintTest('modelName', '=', None) - for record in geometryData: + gridData = self._runConstraintTest('modelName', '=', None) + for record in gridData: self.assertIsNone(record.getAttribute('modelName')) def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('modelName', '!=', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '!=', 'Fcst') + for record in gridData: self.assertNotEqual(record.getAttribute('modelName'), 'Fcst') def testGetDataWithNotEqualsNone(self): - geometryData = self._runConstraintTest('modelName', '!=', None) - for record in geometryData: + gridData = self._runConstraintTest('modelName', '!=', None) + for record in gridData: self.assertIsNotNone(record.getAttribute('modelName')) def testGetDataWithGreaterThan(self): - geometryData = self._runConstraintTest('modelName', '>', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '>', 'Fcst') + for record in gridData: self.assertGreater(record.getAttribute('modelName'), 'Fcst') def testGetDataWithLessThan(self): - geometryData = self._runConstraintTest('modelName', '<', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '<', 'Fcst') + for record in gridData: self.assertLess(record.getAttribute('modelName'), 'Fcst') def testGetDataWithGreaterThanEquals(self): - geometryData = self._runConstraintTest('modelName', '>=', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '>=', 'Fcst') + for record in gridData: self.assertGreaterEqual(record.getAttribute('modelName'), 'Fcst') def testGetDataWithLessThanEquals(self): - geometryData = self._runConstraintTest('modelName', '<=', 'Fcst') - for record in geometryData: + gridData = self._runConstraintTest('modelName', '<=', 'Fcst') + for record in gridData: self.assertLessEqual(record.getAttribute('modelName'), 'Fcst') def testGetDataWithInTuple(self): collection = ('Fcst', 'SAT') - geometryData = self._runConstraintTest('modelName', 'in', collection) - for record in geometryData: + gridData = self._runConstraintTest('modelName', 'in', collection) + for record in gridData: self.assertIn(record.getAttribute('modelName'), collection) def testGetDataWithInList(self): collection = ['Fcst', 'SAT'] - geometryData = self._runConstraintTest('modelName', 'in', collection) - for record in geometryData: + gridData = self._runConstraintTest('modelName', 'in', collection) + for record in gridData: self.assertIn(record.getAttribute('modelName'), collection) def testGetDataWithInGenerator(self): collection = ('Fcst', 'SAT') generator = (item for item in collection) - geometryData = self._runConstraintTest('modelName', 'in', generator) - for record in geometryData: + gridData = self._runConstraintTest('modelName', 'in', generator) + for record in gridData: self.assertIn(record.getAttribute('modelName'), collection) def testGetDataWithNotInList(self): collection = ('Fcst', 'SAT') - geometryData = self._runConstraintTest('modelName', 'not in', collection) - for record in geometryData: + gridData = self._runConstraintTest('modelName', 'not in', collection) + for record in gridData: self.assertNotIn(record.getAttribute('modelName'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): @@ -200,4 +234,4 @@ class GfeTestCase(baseDafTestCase.DafTestCase): def testGetDataWithEmptyInConstraintThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('modelName', 'in', []) \ No newline at end of file + self._runConstraintTest('modelName', 'in', []) diff --git a/pythonPackages/ufpy/test/dafTests/testGrid.py b/pythonPackages/ufpy/test/dafTests/testGrid.py index e20d2e5403..34e8f72bf6 100644 --- a/pythonPackages/ufpy/test/dafTests/testGrid.py +++ b/pythonPackages/ufpy/test/dafTests/testGrid.py @@ -21,10 +21,11 @@ from __future__ import print_function from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint from shapely.geometry import box, Point -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException import baseDafTestCase +import params import unittest # @@ -44,6 +45,9 @@ import unittest # 10/13/16 5942 bsteffen Test envelopes # 11/08/16 5985 tgurney Skip certain tests when no # data is available +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Skip envelope test when no +# data is available # @@ -54,8 +58,6 @@ class GridTestCase(baseDafTestCase.DafTestCase): model = 'GFS160' - envelope = box(-97.0, 41.0, -96.0, 42.0) - def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier('info.datasetId', self.model) @@ -110,18 +112,18 @@ class GridTestCase(baseDafTestCase.DafTestCase): req.addIdentifier('info.datasetId', self.model) req.setLevels('2FHAG') req.setParameters('T') - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) gridData = self.runGridDataTest(req) - if not gridData: - raise unittest.SkipTest('no data available') + if len(gridData) == 0: + raise unittest.SkipTest("No data available") lons, lats = gridData[0].getLatLonCoords() lons = lons.reshape(-1) lats = lats.reshape(-1) # Ensure all points are within one degree of the original box # to allow slight margin of error for reprojection distortion. - testEnv = box(self.envelope.bounds[0] - 1, self.envelope.bounds[1] - 1, - self.envelope.bounds[2] + 1, self.envelope.bounds[3] + 1 ) + testEnv = box(params.ENVELOPE.bounds[0] - 1, params.ENVELOPE.bounds[1] - 1, + params.ENVELOPE.bounds[2] + 1, params.ENVELOPE.bounds[3] + 1 ) for i in range(len(lons)): self.assertTrue(testEnv.contains(Point(lons[i], lats[i]))) @@ -283,4 +285,4 @@ class GridTestCase(baseDafTestCase.DafTestCase): with self.assertRaises(ThriftRequestException) as cm: self.runGridDataTest(req) self.assertIn('IncompatibleRequestException', str(cm.exception)) - self.assertIn('info.level.masterLevel.name', str(cm.exception)) \ No newline at end of file + self.assertIn('info.level.masterLevel.name', str(cm.exception)) diff --git a/pythonPackages/ufpy/test/dafTests/testHydro.py b/pythonPackages/ufpy/test/dafTests/testHydro.py index c49b8fd7bf..8f1accef88 100644 --- a/pythonPackages/ufpy/test/dafTests/testHydro.py +++ b/pythonPackages/ufpy/test/dafTests/testHydro.py @@ -20,8 +20,8 @@ from __future__ import print_function import datetime -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint from dynamicserialize.dstypes.com.raytheon.uf.common.time import TimeRange diff --git a/pythonPackages/ufpy/test/dafTests/testLdadMesonet.py b/pythonPackages/ufpy/test/dafTests/testLdadMesonet.py index 5f6da0d268..38e3dd9cd0 100644 --- a/pythonPackages/ufpy/test/dafTests/testLdadMesonet.py +++ b/pythonPackages/ufpy/test/dafTests/testLdadMesonet.py @@ -20,7 +20,7 @@ from __future__ import print_function from shapely.geometry import Polygon -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase import unittest diff --git a/pythonPackages/ufpy/test/dafTests/testMaps.py b/pythonPackages/ufpy/test/dafTests/testMaps.py index 0b9f716be7..afe1554a01 100644 --- a/pythonPackages/ufpy/test/dafTests/testMaps.py +++ b/pythonPackages/ufpy/test/dafTests/testMaps.py @@ -20,8 +20,8 @@ from __future__ import print_function from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException import baseDafTestCase import unittest @@ -40,6 +40,7 @@ import unittest # 06/13/16 5574 mapeters Add advanced query tests # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN +# 01/06/17 5981 tgurney Do not check data times # # @@ -71,7 +72,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase): req.setLocationNames('OAX') req.addIdentifier('cwa', 'OAX') req.setParameters('countyname', 'state', 'fips') - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -104,22 +105,6 @@ class MapsTestCase(baseDafTestCase.DafTestCase): with self.assertRaises(ThriftRequestException): idValues = DAL.getIdentifierValues(req, 'state') - @unittest.skip('avoid EDEX error') - def testGetColumnIdValuesWithNonexistentTableThrowsException(self): - req = DAL.newDataRequest(self.datatype) - req.addIdentifier('table', 'mapdata.nonexistentjunk') - req.addIdentifier('geomField', 'the_geom') - with self.assertRaises(ThriftRequestException): - idValues = DAL.getIdentifierValues(req, 'state') - - @unittest.skip('avoid EDEX error') - def testGetNonexistentColumnIdValuesThrowsException(self): - req = DAL.newDataRequest(self.datatype) - req.addIdentifier('table', 'mapdata.county') - req.addIdentifier('geomField', 'the_geom') - with self.assertRaises(ThriftRequestException): - idValues = DAL.getIdentifierValues(req, 'nonexistentjunk') - def testGetInvalidIdentifierValuesThrowsException(self): self.runInvalidIdValuesTest() @@ -134,7 +119,7 @@ class MapsTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('state', 'reservoir', 'area_sq_mi') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('state', '=', 'NE') diff --git a/pythonPackages/ufpy/test/dafTests/testModelSounding.py b/pythonPackages/ufpy/test/dafTests/testModelSounding.py index ac8fb79832..c021fa510d 100644 --- a/pythonPackages/ufpy/test/dafTests/testModelSounding.py +++ b/pythonPackages/ufpy/test/dafTests/testModelSounding.py @@ -19,10 +19,11 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -40,6 +41,9 @@ import unittest # 06/30/16 5725 tgurney Add test for NOT IN # 11/10/16 5985 tgurney Mark expected failures prior # to 17.3.1 +# 12/07/16 5981 tgurney Parameterize +# 12/19/16 5981 tgurney Remove pre-17.3 expected fails +# 12/20/16 5981 tgurney Add envelope test # # @@ -51,31 +55,25 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): def testGetAvailableParameters(self): req = DAL.newDataRequest(self.datatype) - self.runParametersTest(req) def testGetAvailableLocations(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - self.runLocationsTest(req) def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - req.setLocationNames("KOMA") - + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) - @unittest.expectedFailure def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) req.addIdentifier("reportType", "ETA") - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") - print("Testing getGeometryData()") - geomData = DAL.getGeometryData(req) print("Number of geometry records: " + str(len(geomData))) print("Sample geometry data:") @@ -84,18 +82,32 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # One dimensional parameters are reported on the 0.0UNKNOWN level. # 2D parameters are reported on MB levels from pressure. if record.getLevel() == "0.0UNKNOWN": - print(" sfcPress=" + record.getString("sfcPress") + record.getUnit("sfcPress"), end="") - print(" temp2=" + record.getString("temp2") + record.getUnit("temp2"), end="") - print(" q2=" + record.getString("q2") + record.getUnit("q2"), end="") - + print(" sfcPress=" + record.getString("sfcPress") + + record.getUnit("sfcPress"), end="") + print(" temp2=" + record.getString("temp2") + + record.getUnit("temp2"), end="") + print(" q2=" + record.getString("q2") + + record.getUnit("q2"), end="") else: - print(" pressure=" + record.getString("pressure") + record.getUnit("pressure"), end="") - print(" temperature=" + record.getString("temperature") + record.getUnit("temperature"), end="") - print(" specHum=" + record.getString("specHum") + record.getUnit("specHum"), end="") + print(" pressure=" + record.getString("pressure") + + record.getUnit("pressure"), end="") + print(" temperature=" + record.getString("temperature") + + record.getUnit("temperature"), end="") + print(" specHum=" + record.getString("specHum") + + record.getUnit("specHum"), end="") print(" geometry=" + str(record.getGeometry())) - print("getGeometryData() complete\n\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.addIdentifier("reportType", "ETA") + req.setEnvelope(params.ENVELOPE) + req.setParameters("temperature", "pressure", "specHum", "sfcPress", "temp2", "q2") + print("Testing getGeometryData()") + data = DAL.getGeometryData(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) + def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) optionalIds = set(DAL.getOptionalIdentifiers(req)) @@ -111,7 +123,7 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.setParameters('dataURI') - req.setLocationNames('KOMA', 'KORD', 'KOFK', 'KLNK') + req.setLocationNames(params.OBS_STATION, 'KORD', 'KOFK', 'KLNK') req.addIdentifier(key, constraint) return self.runGeometryDataTest(req) @@ -123,13 +135,11 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # # Can also eyeball the number of returned records. - @unittest.expectedFailure def testGetDataWithEqualsString(self): geometryData = self._runConstraintTest('reportType', '=', 'ETA') for record in geometryData: self.assertIn('/ETA/', record.getString('dataURI')) - @unittest.expectedFailure def testGetDataWithEqualsUnicode(self): geometryData = self._runConstraintTest('reportType', '=', u'ETA') for record in geometryData: @@ -137,37 +147,29 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): # No numeric tests since no numeric identifiers are available. - @unittest.expectedFailure def testGetDataWithEqualsNone(self): geometryData = self._runConstraintTest('reportType', '=', None) - @unittest.expectedFailure def testGetDataWithNotEquals(self): geometryData = self._runConstraintTest('reportType', '!=', 'ETA') for record in geometryData: self.assertNotIn('/ETA/', record.getString('dataURI')) - @unittest.expectedFailure def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('reportType', '!=', None) - @unittest.expectedFailure def testGetDataWithGreaterThan(self): geometryData = self._runConstraintTest('reportType', '>', 'ETA') - @unittest.expectedFailure def testGetDataWithLessThan(self): geometryData = self._runConstraintTest('reportType', '<', 'ETA') - @unittest.expectedFailure def testGetDataWithGreaterThanEquals(self): geometryData = self._runConstraintTest('reportType', '>=', 'ETA') - @unittest.expectedFailure def testGetDataWithLessThanEquals(self): geometryData = self._runConstraintTest('reportType', '<=', 'ETA') - @unittest.expectedFailure def testGetDataWithInTuple(self): collection = ('ETA', 'GFS') geometryData = self._runConstraintTest('reportType', 'in', collection) @@ -175,7 +177,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): dataURI = record.getString('dataURI') self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI) - @unittest.expectedFailure def testGetDataWithInList(self): collection = ['ETA', 'GFS'] geometryData = self._runConstraintTest('reportType', 'in', collection) @@ -183,7 +184,6 @@ class ModelSoundingTestCase(baseDafTestCase.DafTestCase): dataURI = record.getString('dataURI') self.assertTrue('/ETA/' in dataURI or '/GFS/' in dataURI) - @unittest.expectedFailure def testGetDataWithInGenerator(self): collection = ('ETA', 'GFS') generator = (item for item in collection) diff --git a/pythonPackages/ufpy/test/dafTests/testObs.py b/pythonPackages/ufpy/test/dafTests/testObs.py index bf97bda146..4fafa1c0cd 100644 --- a/pythonPackages/ufpy/test/dafTests/testObs.py +++ b/pythonPackages/ufpy/test/dafTests/testObs.py @@ -19,10 +19,11 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -38,6 +39,8 @@ import unittest # 06/09/16 5587 bsteffen Add getIdentifierValues tests # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test # # @@ -57,14 +60,22 @@ class ObsTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.setParameters("temperature", "seaLevelPress", "dewpoint") - self.runGeometryDataTest(req) + data = self.runGeometryDataTest(req) + + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setParameters("temperature", "seaLevelPress", "dewpoint") + data = self.runGeometryDataTest(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) def testGetIdentifierValues(self): req = DAL.newDataRequest(self.datatype) @@ -81,7 +92,7 @@ class ObsTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) constraint = RequestConstraint.new(operator, value) req.setParameters("temperature", "reportType") - req.setLocationNames("KOMA") + req.setLocationNames(params.OBS_STATION) req.addIdentifier(key, constraint) return self.runGeometryDataTest(req) diff --git a/pythonPackages/ufpy/test/dafTests/testPirep.py b/pythonPackages/ufpy/test/dafTests/testPirep.py index 876a378a67..d2077c4187 100644 --- a/pythonPackages/ufpy/test/dafTests/testPirep.py +++ b/pythonPackages/ufpy/test/dafTests/testPirep.py @@ -19,9 +19,10 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase +import params import unittest # @@ -34,6 +35,8 @@ import unittest # 01/19/16 4795 mapeters Initial Creation. # 04/11/16 5548 tgurney Cleanup # 04/18/16 5548 tgurney More cleanup +# 12/07/16 5981 tgurney Parameterize +# 12/20/16 5981 tgurney Add envelope test # # @@ -53,16 +56,14 @@ class PirepTestCase(baseDafTestCase.DafTestCase): def testGetAvailableTimes(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames('OMA') + req.setLocationNames(params.AIRPORT) self.runTimesTest(req) def testGetGeometryData(self): req = DAL.newDataRequest(self.datatype) - req.setLocationNames('OMA') + req.setLocationNames(params.AIRPORT) req.setParameters("temperature", "windSpeed", "hazardType", "turbType") - print("Testing getGeometryData()") - geomData = DAL.getGeometryData(req) self.assertIsNotNone(geomData) print("Number of geometry records: " + str(len(geomData))) @@ -78,6 +79,13 @@ class PirepTestCase(baseDafTestCase.DafTestCase): print(" hazardType=" + record.getString("hazardType"), end="") print(" turbType=" + record.getString("turbType"), end="") print(" geometry=", record.getGeometry()) - print("getGeometryData() complete\n") + def testGetGeometryDataWithEnvelope(self): + req = DAL.newDataRequest(self.datatype) + req.setParameters("temperature", "windSpeed", "hazardType", "turbType") + req.setEnvelope(params.ENVELOPE) + print("Testing getGeometryData()") + data = DAL.getGeometryData(req) + for item in data: + self.assertTrue(params.ENVELOPE.contains(item.getGeometry())) diff --git a/pythonPackages/ufpy/test/dafTests/testPracticeWarning.py b/pythonPackages/ufpy/test/dafTests/testPracticeWarning.py index 90dee746db..2ee820cccf 100644 --- a/pythonPackages/ufpy/test/dafTests/testPracticeWarning.py +++ b/pythonPackages/ufpy/test/dafTests/testPracticeWarning.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase import testWarning diff --git a/pythonPackages/ufpy/test/dafTests/testProfiler.py b/pythonPackages/ufpy/test/dafTests/testProfiler.py index a05f6d9a8b..b249836748 100644 --- a/pythonPackages/ufpy/test/dafTests/testProfiler.py +++ b/pythonPackages/ufpy/test/dafTests/testProfiler.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL import baseDafTestCase import unittest diff --git a/pythonPackages/ufpy/test/dafTests/testRadarGraphics.py b/pythonPackages/ufpy/test/dafTests/testRadarGraphics.py new file mode 100644 index 0000000000..0cb903d916 --- /dev/null +++ b/pythonPackages/ufpy/test/dafTests/testRadarGraphics.py @@ -0,0 +1,95 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +import unittest + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint +from ufpy.dataaccess import DataAccessLayer as DAL + +import baseRadarTestCase +import params + + +# +# Test DAF support for radar graphics data +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/16 2671 tgurney Initial creation. +# 08/31/16 2671 tgurney Add mesocyclone +# 09/08/16 2671 tgurney Add storm track +# 09/27/16 2671 tgurney Add hail index +# 09/30/16 2671 tgurney Add TVS +# 12/07/16 5981 tgurney Parameterize +# 12/19/16 5981 tgurney Do not check data times on +# returned data +# +# +class RadarGraphicsTestCase(baseRadarTestCase.BaseRadarTestCase): + """Test DAF support for radar data""" + + datatype = 'radar' + + def runConstraintTest(self, key, operator, value): + req = DAL.newDataRequest(self.datatype) + constraint = RequestConstraint.new(operator, value) + req.addIdentifier(key, constraint) + req.setParameters('166') + # TODO: Cannot check datatimes on the result because the times returned + # by getAvailableTimes have level = -1.0, while the time on the actual + # data has the correct level set (>= 0.0). + return self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataMeltingLayer(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('166') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataMesocyclone(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('141') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataStormTrack(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('58') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataHailIndex(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('59') + self.runGeometryDataTest(req, checkDataTimes=False) + + def testGetGeometryDataTVS(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters('61') + self.runGeometryDataTest(req, checkDataTimes=False) diff --git a/pythonPackages/ufpy/test/dafTests/testRadarGrid.py b/pythonPackages/ufpy/test/dafTests/testRadarGrid.py new file mode 100644 index 0000000000..ed82f849ab --- /dev/null +++ b/pythonPackages/ufpy/test/dafTests/testRadarGrid.py @@ -0,0 +1,61 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +from ufpy.dataaccess import DataAccessLayer as DAL +from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint + +import baseRadarTestCase +import params +import unittest + +# +# Test DAF support for radar grid data +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 08/25/16 2671 tgurney Initial creation +# +# + + +class RadarTestCase(baseRadarTestCase.BaseRadarTestCase): + """Test DAF support for radar data""" + + datatype = 'radar' + + parameterList = ['94'] + + def runConstraintTest(self, key, operator, value): + req = DAL.newDataRequest(self.datatype) + constraint = RequestConstraint.new(operator, value) + req.addIdentifier(key, constraint) + req.setParameters(*self.parameterList) + # Don't test shapes since they may differ. + return self.runGridDataTest(req, testSameShape=False) + + def testGetGridData(self): + req = DAL.newDataRequest(self.datatype) + req.setEnvelope(params.ENVELOPE) + req.setLocationNames(self.radarLoc) + req.setParameters(*self.parameterList) + # Don't test shapes since they may differ. + self.runGridDataTest(req, testSameShape=False) diff --git a/pythonPackages/ufpy/test/dafTests/testRadarSpatial.py b/pythonPackages/ufpy/test/dafTests/testRadarSpatial.py index 039b3d2e8f..354eb4b580 100644 --- a/pythonPackages/ufpy/test/dafTests/testRadarSpatial.py +++ b/pythonPackages/ufpy/test/dafTests/testRadarSpatial.py @@ -20,10 +20,11 @@ from __future__ import print_function from shapely.geometry import box -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase +import params import unittest # @@ -41,6 +42,8 @@ import unittest # superclass # 06/13/16 5574 tgurney Add advanced query tests # 06/30/16 5725 tgurney Add test for NOT IN +# 12/07/16 5981 tgurney Parameterize +# 01/06/17 5981 tgurney Do not check data times # # @@ -50,14 +53,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): datatype = "radar_spatial" - envelope = box(-97.0, 41.0, -96.0, 42.0) - """ - Default request area (box around KOAX) - """ - def testGetAvailableLocations(self): req = DAL.newDataRequest(self.datatype) - req.setEnvelope(self.envelope) + req.setEnvelope(params.ENVELOPE) self.runLocationsTest(req) def testGetAvailableParameters(self): @@ -71,7 +69,7 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): req = DAL.newDataRequest(self.datatype) req.setLocationNames("TORD", "TMDW") req.setParameters("wfo_id", "name", "elevmeter") - self.runGeometryDataTest(req) + self.runGeometryDataTest(req, checkDataTimes=False) def testRequestingTimesThrowsTimeAgnosticDataException(self): req = DAL.newDataRequest(self.datatype) @@ -82,17 +80,17 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): constraint = RequestConstraint.new(operator, value) req.addIdentifier(key, constraint) req.setParameters('elevmeter', 'eqp_elv', 'wfo_id', 'immutablex') - return self.runGeometryDataTest(req) + return self.runGeometryDataTest(req, checkDataTimes=False) def testGetDataWithEqualsString(self): - geometryData = self._runConstraintTest('wfo_id', '=', 'OAX') + geometryData = self._runConstraintTest('wfo_id', '=', params.SITE_ID) for record in geometryData: - self.assertEqual(record.getString('wfo_id'), 'OAX') + self.assertEqual(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithEqualsUnicode(self): - geometryData = self._runConstraintTest('wfo_id', '=', u'OAX') + geometryData = self._runConstraintTest('wfo_id', '=', unicode(params.SITE_ID)) for record in geometryData: - self.assertEqual(record.getString('wfo_id'), 'OAX') + self.assertEqual(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithEqualsInt(self): geometryData = self._runConstraintTest('immutablex', '=', 57) @@ -115,9 +113,9 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): self.assertEqual(record.getType('wfo_id'), 'NULL') def testGetDataWithNotEquals(self): - geometryData = self._runConstraintTest('wfo_id', '!=', 'OAX') + geometryData = self._runConstraintTest('wfo_id', '!=', params.SITE_ID) for record in geometryData: - self.assertNotEquals(record.getString('wfo_id'), 'OAX') + self.assertNotEquals(record.getString('wfo_id'), params.SITE_ID) def testGetDataWithNotEqualsNone(self): geometryData = self._runConstraintTest('wfo_id', '!=', None) @@ -145,33 +143,33 @@ class RadarSpatialTestCase(baseDafTestCase.DafTestCase): self.assertLessEqual(record.getNumber('eqp_elv'), 138) def testGetDataWithInTuple(self): - collection = ('OAX', 'GID') + collection = (params.SITE_ID, 'GID') geometryData = self._runConstraintTest('wfo_id', 'in', collection) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithInList(self): - collection = ['OAX', 'GID'] + collection = [params.SITE_ID, 'GID'] geometryData = self._runConstraintTest('wfo_id', 'in', collection) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithInGenerator(self): - collection = ('OAX', 'GID') + collection = (params.SITE_ID, 'GID') generator = (item for item in collection) geometryData = self._runConstraintTest('wfo_id', 'in', generator) for record in geometryData: self.assertIn(record.getString('wfo_id'), collection) def testGetDataWithNotInList(self): - collection = ['OAX', 'GID'] + collection = [params.SITE_ID, 'GID'] geometryData = self._runConstraintTest('wfo_id', 'not in', collection) for record in geometryData: self.assertNotIn(record.getString('wfo_id'), collection) def testGetDataWithInvalidConstraintTypeThrowsException(self): with self.assertRaises(ValueError): - self._runConstraintTest('wfo_id', 'junk', 'OAX') + self._runConstraintTest('wfo_id', 'junk', params.SITE_ID) def testGetDataWithInvalidConstraintValueThrowsException(self): with self.assertRaises(TypeError): diff --git a/pythonPackages/ufpy/test/dafTests/testSatellite.py b/pythonPackages/ufpy/test/dafTests/testSatellite.py index 900ab657ca..3e42b1d2aa 100644 --- a/pythonPackages/ufpy/test/dafTests/testSatellite.py +++ b/pythonPackages/ufpy/test/dafTests/testSatellite.py @@ -20,7 +20,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase diff --git a/pythonPackages/ufpy/test/dafTests/testSfcObs.py b/pythonPackages/ufpy/test/dafTests/testSfcObs.py index 95c92c5f0c..4387eadee7 100644 --- a/pythonPackages/ufpy/test/dafTests/testSfcObs.py +++ b/pythonPackages/ufpy/test/dafTests/testSfcObs.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase diff --git a/pythonPackages/ufpy/test/dafTests/testTopo.py b/pythonPackages/ufpy/test/dafTests/testTopo.py index 1ed41316be..d0dcd36865 100644 --- a/pythonPackages/ufpy/test/dafTests/testTopo.py +++ b/pythonPackages/ufpy/test/dafTests/testTopo.py @@ -19,8 +19,8 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL -from awips.ThriftClient import ThriftRequestException +from ufpy.dataaccess import DataAccessLayer as DAL +from ufpy.ThriftClient import ThriftRequestException import baseDafTestCase import shapely.geometry @@ -39,7 +39,7 @@ import unittest # 05/26/16 5587 tgurney Add test for # getIdentifierValues() # 06/01/16 5587 tgurney Update testGetIdentifierValues -# +# 07/18/17 6253 randerso Removed referenced to GMTED # @@ -61,7 +61,7 @@ class TopoTestCase(baseDafTestCase.DafTestCase): print("Sample grid data shape:\n" + str(gridData[0].getRawData().shape) + "\n") print("Sample grid data:\n" + str(gridData[0].getRawData()) + "\n") - for topoFile in ["gmted2010", "gtopo30"]: + for topoFile in ["gtopo30"]: print("\n" + topoFile) req.addIdentifier("topoFile", topoFile) gridData = DAL.getGridData(req) diff --git a/pythonPackages/ufpy/test/dafTests/testWarning.py b/pythonPackages/ufpy/test/dafTests/testWarning.py index fd0ef4e3cd..d1ece0c115 100644 --- a/pythonPackages/ufpy/test/dafTests/testWarning.py +++ b/pythonPackages/ufpy/test/dafTests/testWarning.py @@ -19,7 +19,7 @@ ## from __future__ import print_function -from awips.dataaccess import DataAccessLayer as DAL +from ufpy.dataaccess import DataAccessLayer as DAL from dynamicserialize.dstypes.com.raytheon.uf.common.dataquery.requests import RequestConstraint import baseDafTestCase @@ -42,6 +42,7 @@ import unittest # 06/13/16 5574 tgurney Fix checks for None # 06/21/16 5548 tgurney Skip tests that cause errors # 06/30/16 5725 tgurney Add test for NOT IN +# 12/12/16 5981 tgurney Improve test performance # # @@ -81,22 +82,19 @@ class WarningTestCase(baseDafTestCase.DafTestCase): self.runGeometryDataTest(req) def testFilterOnLocationName(self): - allRecordsCount = len(self._getAllRecords()) allLocationNames = self._getLocationNames() - if allRecordsCount == 0: + if len(allLocationNames) == 0: errmsg = "No {0} data exists on {1}. Try again with {0} data." raise unittest.SkipTest(errmsg.format(self.datatype, DAL.THRIFT_HOST)) - if len(allLocationNames) != 1: - testCount = 3 # number of different location names to test - for locationName in allLocationNames[:testCount]: - req = DAL.newDataRequest() - req.setDatatype(self.datatype) - req.setParameters('id') - req.setLocationNames(locationName) - geomData = DAL.getGeometryData(req) - self.assertLess(len(geomData), allRecordsCount) - for geom in geomData: - self.assertEqual(geom.getLocationName(), locationName) + testCount = 3 # number of different location names to test + for locationName in allLocationNames[:testCount]: + req = DAL.newDataRequest() + req.setDatatype(self.datatype) + req.setParameters('id') + req.setLocationNames(locationName) + geomData = DAL.getGeometryData(req) + for geom in geomData: + self.assertEqual(geom.getLocationName(), locationName) def testFilterOnNonexistentLocationReturnsEmpty(self): req = DAL.newDataRequest() diff --git a/pythonPackages/ufpy/test/testQpidTimeToLive.py b/pythonPackages/ufpy/test/testQpidTimeToLive.py index d4fbc1b699..ce3f074703 100644 --- a/pythonPackages/ufpy/test/testQpidTimeToLive.py +++ b/pythonPackages/ufpy/test/testQpidTimeToLive.py @@ -52,7 +52,7 @@ class ListenThread(threading.Thread): threading.Thread.__init__(self) def run(self): - from awips import QpidSubscriber + from ufpy import QpidSubscriber self.qs = QpidSubscriber.QpidSubscriber(self.hostname, self.portNumber, True) self.qs.topicSubscribe(self.topicName, self.receivedMessage)