From 4e7b1f7721de9a72da15833ba3e84e0d5095a076 Mon Sep 17 00:00:00 2001 From: ucar-tmeyer Date: Wed, 27 Sep 2023 13:04:01 +0000 Subject: [PATCH] Added back in the ldad/mesonet plugins so that we can decode the RAWS mesonet data. -Also updated the Surface Obs menu to have a few menu options to load the RAWS data --- .../localization/bundles/LDADMesoPlot.xml | 22 - .../localization/menus/obs/baseSurface.xml | 42 +- .../com.raytheon.edex.plugin.ldad/.classpath | 7 + .../com.raytheon.edex.plugin.ldad/.project | 28 + .../META-INF/MANIFEST.MF | 16 + .../build.properties | 6 + .../com.raytheon.edex.plugin.ldad.ecl | 0 .../res/spring/ldad-common.xml | 17 + .../edex/plugin/ldad/LdadDecoder.java | 575 ++++++ .../edex/plugin/ldad/common/DecodedData.java | 67 + .../edex/plugin/ldad/common/LdadDataType.java | 59 + .../edex/plugin/ldad/common/LdadField.java | 56 + .../base/ldad/ldadTimeZoneMap.txt | 36 + .../common_static/base/ldad/ldadUnitsMap.txt | 305 +++ .../.classpath | 7 + .../.project | 28 + .../META-INF/MANIFEST.MF | 13 + .../build.properties | 4 + .../uf/common/dataplugin/ldad/LdadRecord.java | 126 ++ .../.classpath | 7 + .../.project | 28 + .../META-INF/MANIFEST.MF | 16 + .../build.properties | 5 + ...theon.uf.common.dataplugin.ldadmesonet.ecl | 0 .../spring/ldadmesonet-common-dataaccess.xml | 13 + .../ldadmesonet/MesonetLdadRecord.java | 1702 +++++++++++++++++ .../.classpath | 7 + .../.project | 28 + .../META-INF/MANIFEST.MF | 15 + .../build.properties | 4 + .../com.raytheon.uf.common.dataplugin.qc.ecl | 0 ...aytheon.uf.common.dataplugin.qc.properties | 1 + .../uf/common/dataplugin/qc/QCRecord.java | 197 ++ .../feature.xml | 41 + .../.classpath | 7 + .../.project | 28 + .../META-INF/MANIFEST.MF | 12 + .../build.properties | 6 + ...om.raytheon.uf.edex.plugin.ldadmesonet.ecl | 0 .../res/pointdata/ldadmesonet.xml | 91 + .../res/pointdata/ldadmesonetdb.xml | 30 + .../res/spring/ldadmesonet-common.xml | 21 + .../res/spring/ldadmesonet-ingest.xml | 45 + .../LdadmesonetPointDataTransform.java | 557 ++++++ .../ldadmesonet/dao/LdadMesonetDao.java | 163 ++ .../base/distribution/ldadmesonet.xml | 29 + .../base/purge/ldadmesonetPurgeRules.xml | 36 + .../com.raytheon.uf.edex.plugin.qc/.classpath | 7 + .../com.raytheon.uf.edex.plugin.qc/.project | 34 + .../.pydevproject | 7 + .../META-INF/MANIFEST.MF | 13 + .../build.properties | 7 + .../com.raytheon.uf.edex.plugin.qc.ecl | 0 .../res/pointdata/pdd/ldadmesonet.xml | 58 + .../res/pointdata/pdd/msas.xml | 61 + .../res/pointdata/qcdb.xml | 29 + .../res/spring/qc-common.xml | 29 + .../res/spring/qc-ingest.xml | 61 + .../resources/qc.properties | 2 + .../raytheon/uf/edex/plugin/qc/QCScanner.java | 370 ++++ .../raytheon/uf/edex/plugin/qc/dao/QCDao.java | 92 + .../uf/edex/plugin/qc/internal/QCPaths.java | 123 ++ .../common_static/base/path/qcPathKeys.xml | 32 + 63 files changed, 5405 insertions(+), 23 deletions(-) create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/.classpath create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/.project create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/build.properties create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/com.raytheon.edex.plugin.ldad.ecl create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/res/spring/ldad-common.xml create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/LdadDecoder.java create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/DecodedData.java create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadDataType.java create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadField.java create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadTimeZoneMap.txt create mode 100644 edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadUnitsMap.txt create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.classpath create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.project create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldad/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldad/build.properties create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldad/src/com/raytheon/uf/common/dataplugin/ldad/LdadRecord.java create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.classpath create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.project create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/build.properties create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/com.raytheon.uf.common.dataplugin.ldadmesonet.ecl create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/res/spring/ldadmesonet-common-dataaccess.xml create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/src/com/raytheon/uf/common/dataplugin/ldadmesonet/MesonetLdadRecord.java create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/.classpath create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/.project create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/build.properties create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/com.raytheon.uf.common.dataplugin.qc.ecl create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/resources/com.raytheon.uf.common.dataplugin.qc.properties create mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.qc/src/com/raytheon/uf/common/dataplugin/qc/QCRecord.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/.classpath create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/.project create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/build.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/com.raytheon.uf.edex.plugin.ldadmesonet.ecl create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonet.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonetdb.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-common.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-ingest.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/LdadmesonetPointDataTransform.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/dao/LdadMesonetDao.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/distribution/ldadmesonet.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/purge/ldadmesonetPurgeRules.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/.classpath create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/.project create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/.pydevproject create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/build.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/com.raytheon.uf.edex.plugin.qc.ecl create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/ldadmesonet.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/msas.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/qcdb.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-common.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-ingest.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/resources/qc.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/QCScanner.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/dao/QCDao.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/internal/QCPaths.java create mode 100644 edexOsgi/com.raytheon.uf.edex.plugin.qc/utility/common_static/base/path/qcPathKeys.xml diff --git a/cave/com.raytheon.viz.pointdata/localization/bundles/LDADMesoPlot.xml b/cave/com.raytheon.viz.pointdata/localization/bundles/LDADMesoPlot.xml index b04fb1d080..69834e2e9b 100644 --- a/cave/com.raytheon.viz.pointdata/localization/bundles/LDADMesoPlot.xml +++ b/cave/com.raytheon.viz.pointdata/localization/bundles/LDADMesoPlot.xml @@ -28,28 +28,6 @@ - - - - - - - PLAN_VIEW - - - - - - basemaps/ldad15.spi - LDAD Stations - - diff --git a/cave/com.raytheon.viz.pointdata/localization/menus/obs/baseSurface.xml b/cave/com.raytheon.viz.pointdata/localization/menus/obs/baseSurface.xml index a0459170ce..f8b611e649 100644 --- a/cave/com.raytheon.viz.pointdata/localization/menus/obs/baseSurface.xml +++ b/cave/com.raytheon.viz.pointdata/localization/menus/obs/baseSurface.xml @@ -8,9 +8,49 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/.classpath b/edexOsgi/com.raytheon.edex.plugin.ldad/.classpath new file mode 100644 index 0000000000..1fa3e6803d --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/.project b/edexOsgi/com.raytheon.edex.plugin.ldad/.project new file mode 100644 index 0000000000..85ace3766b --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/.project @@ -0,0 +1,28 @@ + + + com.raytheon.edex.plugin.ldad + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.edex.plugin.ldad/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..aa18849bfc --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/META-INF/MANIFEST.MF @@ -0,0 +1,16 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Ldad Plug-in +Bundle-SymbolicName: com.raytheon.edex.plugin.ldad +Bundle-Version: 1.18.1.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-11 +Export-Package: com.raytheon.edex.plugin.ldad.common +Require-Bundle: com.raytheon.uf.common.dataplugin.ldad, + com.raytheon.uf.common.pointdata, + com.raytheon.edex.common, + com.raytheon.uf.common.localization, + javax.measure, + org.slf4j, + javax.xml.bind +Import-Package: com.raytheon.uf.common.status diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/build.properties b/edexOsgi/com.raytheon.edex.plugin.ldad/build.properties new file mode 100644 index 0000000000..20a5272307 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/build.properties @@ -0,0 +1,6 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + utility/,\ + .,\ + res/ diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/com.raytheon.edex.plugin.ldad.ecl b/edexOsgi/com.raytheon.edex.plugin.ldad/com.raytheon.edex.plugin.ldad.ecl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/res/spring/ldad-common.xml b/edexOsgi/com.raytheon.edex.plugin.ldad/res/spring/ldad-common.xml new file mode 100644 index 0000000000..8cad61a9ef --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/res/spring/ldad-common.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/LdadDecoder.java b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/LdadDecoder.java new file mode 100644 index 0000000000..8d663b312b --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/LdadDecoder.java @@ -0,0 +1,575 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ + +package com.raytheon.edex.plugin.ldad; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.ParseException; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.time.DateTimeException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; + +import javax.measure.IncommensurableException; +import javax.measure.UnconvertibleException; +import javax.measure.Unit; +import javax.xml.bind.JAXBException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.raytheon.edex.plugin.ldad.common.DecodedData; +import com.raytheon.edex.plugin.ldad.common.LdadField; +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.ldad.LdadRecord; +import com.raytheon.uf.common.localization.ILocalizationFile; +import com.raytheon.uf.common.localization.IPathManager; +import com.raytheon.uf.common.localization.LocalizationUtil; +import com.raytheon.uf.common.localization.PathManagerFactory; +import com.raytheon.uf.common.localization.exception.LocalizationException; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.common.serialization.JAXBManager; +import com.raytheon.uf.common.serialization.SerializationException; +import com.raytheon.uf.common.time.DataTime; + +import net.sf.cglib.beans.BeanMap; +import tec.uom.se.format.SimpleUnitFormat; + +/** + * Decoder implementation for ldadmesonet plugin. + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- --------------------------------------------
+ * Sep 04, 2009           vkorolev  Initial creation
+ * May 15, 2013  1869     bsteffen  Remove DataURI column from ldadmesonet.
+ * Aug 30, 2013  2298     rjpeter   Make getPluginName abstract
+ * Jul 23, 2014  3410     bclement  location changed to floats
+ * Aug 15, 2014  3530     bclement  no longer extends AbstractDecoder
+ * Jul 08, 2016  5744     mapeters  Config file moved from edex_static to
+ *                                  common_static
+ * Dec 18, 2017  6897     tgurney   Handle date value in a Double field
+ * Mar 06, 2018  6851     randerso  Added lookup table for time zones. Lots of
+ *                                  code cleanup.
+ * May 09, 2018  7288     randerso  Use stationId if available
+ * Apr 15, 2019  7596     lsingh    Updated units framework to JSR-363.
+ *                                  Handled unit conversion.
+ *
+ *
+ * + * @author vkorolev + */ + +public class LdadDecoder { + private static final Logger logger = LoggerFactory + .getLogger(LdadDecoder.class); + + private static final String DATE_TIME_STRING_UNITS = "DATE_TIME_STRING"; + + private static final String OBSERVATION_TIME_KEY = "observationTime"; + + private static final String TIMEZONE_KEY = "_tz"; + + private static final String PROVIDER_ID_KEY = "providerId"; + + private static final String STATION_ID_KEY = "stationId"; + + private static final String LATITUDE_KEY = "_lat"; + + private static final String LONGITUDE_KEY = "_lon"; + + private static final String ELEVATION_KEY = "_elev"; + + private static final String BAD_PROPERTY_FMT = "NumberFormatException setting property %s.%s(%s %s)"; + + private static final String DATE_FORMAT = "yy/MM/dd HH:mm:ss"; + + private static final ThreadLocal DateFormatter = new ThreadLocal() { + @Override + protected SimpleDateFormat initialValue() { + return new SimpleDateFormat(DATE_FORMAT); + } + }; + + private static final TimeZone UTC = TimeZone.getTimeZone("UTC"); + + private static JAXBManager jaxb; + + private static Properties ldadUnitsMap = new Properties(); + + private static Date ldadUnitsDate = new Date(0); + + private static String ldadUnitsChecksum = ILocalizationFile.NON_EXISTENT_CHECKSUM; + + private static Properties ldadTimeZoneMap = new Properties(); + + private static Date ldadTimeZoneDate = new Date(0); + + private static String ldadTimeZoneChecksum = ILocalizationFile.NON_EXISTENT_CHECKSUM; + + private final Class recordClass; + + private final String storageType; + + /** + * Constructor + * + * @param recordClass + * LdadRecord subclass to be decoded + * @param storageType + * storageType of LDAD files to accept. All others are ignored. + * @throws JAXBException + */ + public LdadDecoder(Class recordClass, + String storageType) throws JAXBException { + this.recordClass = recordClass; + this.storageType = storageType; + synchronized (LdadDecoder.class) { + if (jaxb == null) { + jaxb = new JAXBManager(DecodedData.class); + } + } + } + + /** + * Decode the raw data in to PluginDataObjects + * + * @param data + * the raw data + * @return the decoded records + */ + public PluginDataObject[] decode(byte[] data) { + PluginDataObject[] retVal = new PluginDataObject[0]; + if (data != null) { + IPathManager pathMgr = PathManagerFactory.getPathManager(); + + String filePath = LocalizationUtil.join("ldad", "ldadUnitsMap.txt"); + ILocalizationFile lf = pathMgr.getStaticLocalizationFile(filePath); + if (lf != null) { + synchronized (ldadUnitsMap) { + // update the units map if necessary + if (loadPropertiesFile(ldadUnitsMap, lf, ldadUnitsDate, + ldadUnitsChecksum)) { + // update properties file info + ldadUnitsDate = lf.getTimeStamp(); + ldadUnitsChecksum = lf.getCheckSum(); + } + } + } + + filePath = LocalizationUtil.join("ldad", "ldadTimeZoneMap.txt"); + lf = pathMgr.getStaticLocalizationFile(filePath); + if (lf != null) { + synchronized (ldadTimeZoneMap) { + // update the time zone map if necessary + if (loadPropertiesFile(ldadTimeZoneMap, lf, + ldadTimeZoneDate, ldadTimeZoneChecksum)) { + // update properties file info + ldadTimeZoneDate = lf.getTimeStamp(); + ldadTimeZoneChecksum = lf.getCheckSum(); + } + } + } + + try { + DecodedData dd = (DecodedData) jaxb.unmarshalFromInputStream( + new ByteArrayInputStream(data)); + + // Storage type separator + String currentFile = dd.fileName; + if (!this.storageType.equals(dd.storageType)) { + logger.warn(String.format( + "LDAD decoder for %s received file %s of type %s. File ignored.", + this.storageType, currentFile, dd.storageType)); + return retVal; + } + + // Header + String missingValue = dd.missingValue; + + // Number of records + int numRecs = dd.fields.get(0).values.size(); + + if (numRecs == 0) { + logger.info("No data in file."); + return retVal; + } + + // Create a map of fields + Map fieldMap = new HashMap<>( + dd.fields.size(), 1.0f); + for (LdadField field : dd.fields) { + fieldMap.put(field.variableName, field); + } + Set keySet = new HashSet<>(fieldMap.keySet()); + + // Check for observation time + if (!fieldMap.containsKey(OBSERVATION_TIME_KEY)) { + logger.error(String.format( + "No observation times present in file %s", + currentFile)); + return retVal; + } + + // Check for lat/lon + if (!fieldMap.containsKey(LATITUDE_KEY) + || !fieldMap.containsKey(LONGITUDE_KEY)) { + logger.error(String.format("No location present in file %s", + currentFile)); + return retVal; + } + + // Assume UTC if no time zone specified in file + if (!fieldMap.containsKey(TIMEZONE_KEY)) { + logger.warn(String.format( + "No time zone specified in file %s, assuming UTC", + currentFile)); + } + + // Loop through records + BeanMap beanMap = BeanMap.create(recordClass.newInstance()); + List records = new ArrayList<>(numRecs); + for (int i = 0; i < numRecs; i++) { + LdadRecord record = recordClass.newInstance(); + + SurfaceObsLocation location = new SurfaceObsLocation(); + record.setDataProvider(dd.provider); + record.setStationType(dd.type); + record.setReportTime(dd.reportTime); + + // Set of all known keys remaining to be processed + keySet.addAll(fieldMap.keySet()); + + // Get time zone if present + TimeZone timeZone = UTC; + if (keySet.contains(TIMEZONE_KEY)) { + String tz = fieldMap.get(TIMEZONE_KEY).values.get(i); + keySet.remove(TIMEZONE_KEY); + + synchronized (ldadTimeZoneMap) { + tz = ldadTimeZoneMap.getProperty(tz, tz); + } + try { + ZoneId zoneId = ZoneId.of(tz); + timeZone = TimeZone.getTimeZone(zoneId); + } catch (DateTimeException e) { + logger.error(String.format( + "Unrecognized time zone: %s in record %d of file %s, assuming UTC", + tz, i, currentFile), e); + } + } + + // Get observation time + LdadField field = fieldMap.get(OBSERVATION_TIME_KEY); + keySet.remove(OBSERVATION_TIME_KEY); + String value = field.values.get(i); + if (missingValue.equals(value)) { + logMissingValue(field, i, currentFile); + continue; + } + + try { + record.setObservationTime(parseDate(value, timeZone)); + } catch (ParseException e) { + logValueError(field, i, currentFile, e); + continue; + } + + // Get location + field = fieldMap.get(LATITUDE_KEY); + keySet.remove(LATITUDE_KEY); + value = field.values.get(i); + if (missingValue.equals(value)) { + logMissingValue(field, i, currentFile); + continue; + } + + float latitude; + try { + latitude = Float.parseFloat(value); + } catch (NumberFormatException e) { + logValueError(field, i, currentFile, e); + continue; + } + + field = fieldMap.get(LONGITUDE_KEY); + keySet.remove(LONGITUDE_KEY); + value = field.values.get(i); + if (missingValue.equals(value)) { + logMissingValue(field, i, currentFile); + continue; + } + + float longitude; + try { + longitude = Float.parseFloat(value); + } catch (NumberFormatException e) { + logValueError(field, i, currentFile, e); + continue; + } + + location.assignLocation(latitude, longitude); + + if (keySet.contains(ELEVATION_KEY)) { + field = fieldMap.get(ELEVATION_KEY); + keySet.remove(ELEVATION_KEY); + value = field.values.get(i); + if (!missingValue.equals(value)) { + try { + // elevation in meter - integer in location + double elevation = Double.parseDouble(value); + location.setElevation( + (int) Math.round(elevation)); + } catch (NumberFormatException e) { + logValueError(field, i, currentFile, e); + } + } + } + + // set station ID to provider ID in case station ID is not + // present + if (keySet.contains(PROVIDER_ID_KEY)) { + field = fieldMap.get(PROVIDER_ID_KEY); + + // leave provider id in key set so providerId field is + // populated in the record + + value = field.values.get(i); + location.setStationId(value); + } + + if (keySet.contains(STATION_ID_KEY)) { + field = fieldMap.get(STATION_ID_KEY); + keySet.remove(STATION_ID_KEY); + value = field.values.get(i); + location.setStationId(value); + } + + // Loop through remaining fields + beanMap.setBean(record); + for (String key : keySet) { + field = fieldMap.get(key); + String name = field.variableName; + if (beanMap.containsKey(name)) { + String units = field.units; + value = field.values.get(i); + if (!missingValue.equals(value)) { + try { + // try setting field via reflection + setProperty(name, beanMap, value, units, + timeZone); + } catch (Throwable e) { + logPropertyError(field, i, currentFile, e); + } + } + } else { + /* + * Some fields are not supported. Perfectly valid + * data can cause this exception so we log it as + * debug + */ + logger.debug(String.format( + "Unrecognized field: %s, will be ignored.", + name)); + } + } + record = (LdadRecord) beanMap.getBean(); + + // DataTime = Observation time + Date ot = record.getObservationTime(); + if (ot != null) { + DataTime dt = new DataTime(ot); + record.setDataTime(dt); + record.setLocation(location); + record.setRawMessage(record.toMessage()); + records.add(record); + } + } + + retVal = records.toArray(new PluginDataObject[records.size()]); + + } catch (SerializationException e) { + logger.error("Unable to unmarshall xml:", e); + } catch (RuntimeException e) { + logger.error("Error decoding ldad mesonet data:" + e); + } catch (InstantiationException | IllegalAccessException e) { + logger.error("Unable to instantiate class: " + + this.recordClass.getName(), e); + } + } + + return retVal; + } + + /** + * Loads properties from a localization file + * + * @param props + * properties instance to be updated + * @param lf + * localization file from which to load properties + * @return true if properties were successfully loaded + */ + private boolean loadPropertiesFile(Properties props, ILocalizationFile lf, + Date lastTimeStamp, String lastChecksum) { + boolean status = false; + if (lf.exists() && (!lastChecksum.equals(lf.getCheckSum()) + || !lastTimeStamp.equals(lf.getTimeStamp()))) { + try (InputStream is = lf.openInputStream()) { + Properties newProps = new Properties(); + newProps.load(is); + props.clear(); + props.putAll(newProps); + status = true; + } catch (LocalizationException | IOException e) { + logger.error("Error loading properites from: " + lf, e); + } + } + return status; + } + + private void logMissingValue(LdadField field, int index, String file) { + logger.error(String.format( + "Missing value in field: %s value: %s with units: %s for file: %s", + field.variableName, field.values.get(index), field.units, + file)); + } + + private void logValueError(LdadField field, int index, String file, + Throwable e) { + logger.error(String.format( + "Invalid value in field: %s value: %s with units: %s for file: %s", + field.variableName, field.values.get(index), field.units, file), + e); + } + + private void logPropertyError(LdadField field, int index, String file, + Throwable e) { + logger.error(String.format( + "Unable to set property %s to value: %s with units: %s for file: %s", + field.variableName, field.values.get(index), field.units, file), + e); + } + + private void setProperty(String name, BeanMap beanMap, String value, + String units, TimeZone timeZone) throws ParseException { + + Object val = null; + boolean abort = false; + Class clazz = beanMap.getPropertyType(name); + + // Type filter + if (String.class == clazz) { + val = value.trim(); + } else if (Calendar.class == clazz) { + val = parseDate(value, timeZone); + } else if (DATE_TIME_STRING_UNITS.equals(units)) { + // String date/time in a Double field, convert to epoch seconds + Date date = parseDate(value, timeZone); + val = date.getTime() / 1000.0; + } else { + + // Get rid of some troublesome data + // TODO: find out what should be done with these values + abort = "B".equals(value); + abort |= "R".equals(value); + abort |= "V".equals(value); + abort |= "NAN0".equals(value); + + if (!abort) { + Double tval = null; + try { + tval = Double.parseDouble(value); + } catch (NumberFormatException nfe) { + String msg = String.format(BAD_PROPERTY_FMT, + beanMap.getBean().getClass().getSimpleName(), name, + clazz.getSimpleName(), value); + logger.error(msg, nfe); + return; + } + synchronized (ldadUnitsMap) { + if (ldadUnitsMap.containsKey(units)) { + String translatedUnit = ldadUnitsMap.getProperty(units, + units); + + Unit inUnit = (Unit) SimpleUnitFormat.getInstance(SimpleUnitFormat.Flavor.ASCII) + .parseObject(translatedUnit, new ParsePosition(0)); + + String propUnit = ldadUnitsMap.getProperty(name); + if (propUnit == null) { + logger.error(String.format( + "No units defined in ldadUnitsMap.txt for property: %s", + name)); + } else { + Unit outUnit = (Unit) SimpleUnitFormat.getInstance(SimpleUnitFormat.Flavor.ASCII) + .parseObject(propUnit, new ParsePosition(0)); + try { + tval = inUnit.getConverterToAny(outUnit) + .convert((tval).doubleValue()); + } catch (IncommensurableException | UnconvertibleException e) { + logger.error(String.format( + "Property[%s] Input unit %s not compatible with Output unit %s", + name, units, outUnit), e); + return; + } + } + } + } + + if (clazz == Integer.class) { + val = tval.intValue(); + } else if (clazz == Short.class) { + val = tval.shortValue(); + } else if (clazz == Float.class) { + val = tval.floatValue(); + } else { + val = tval; + } + } + } + if (!abort) { + beanMap.put(name, val); + } + } + + private Date parseDate(String dateTime, TimeZone timeZone) + throws ParseException { + SimpleDateFormat sdf = DateFormatter.get(); + sdf.setTimeZone(timeZone); + + Date date = sdf.parse(dateTime); + return date; + } +} diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/DecodedData.java b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/DecodedData.java new file mode 100644 index 0000000000..54bb36fc4b --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/DecodedData.java @@ -0,0 +1,67 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.ldad.common; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +/** + * Decoded LDAD XML data structure. + * + * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * 08/17/09					dfriedman	Initial creation
+ * 
+ *
+ * + * @author dfriedman + * @version 1.0 + */ + +@XmlRootElement +public class DecodedData { + @XmlAttribute + public String fileName; // original file name + @XmlAttribute + public String storageType; // "mesonet", "hydro", etc. + @XmlAttribute + public String type; // data type name or "msas_qc." + @XmlAttribute + public String root; // original data type name + @XmlAttribute + public String source; // same as type? + @XmlAttribute + public String provider; // data type name? or "MSAS_QC" + @XmlAttribute + public String missingValue; + @XmlAttribute + public long reportTime; // Report time in seconds since 1/1/1970 + + @XmlElement(name="field") + public List fields = new ArrayList(); + + +} diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadDataType.java b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadDataType.java new file mode 100644 index 0000000000..71403f44f5 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadDataType.java @@ -0,0 +1,59 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.ldad.common; + +/** + * Decoded LDAD data types. + * + * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * 08/17/09					dfriedman	Initial creation
+ * 
+ *
+ * + * @author dfriedman + * @version 1.0 + */ + +public enum LdadDataType { + STRING(0), + DATE_TIME(1), + SHORT(2), INT(3), LONG(4), + FLOAT(5), DOUBLE(6); + + private int id; + private LdadDataType(int id) { + this.id = id; + } + public boolean isNumeric() { + return this != STRING && this != DATE_TIME; + } + public static LdadDataType fromId(int id) { + for (LdadDataType t : LdadDataType.values()) + if (t.getId() == id) + return t; + return null; + } + private int getId() { + return id; + } +} diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadField.java b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadField.java new file mode 100644 index 0000000000..a761735583 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/src/com/raytheon/edex/plugin/ldad/common/LdadField.java @@ -0,0 +1,56 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.edex.plugin.ldad.common; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlType; + +/** + * Decoded LDAD XML data structure. + * + * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * 08/17/09					dfriedman	Initial creation
+ * 
+ *
+ * + * @author dfriedman + * @version 1.0 + */ + +@XmlType(name="field") +public class LdadField { + @XmlAttribute + public String variableName; + @XmlAttribute + public String units; + @XmlAttribute + public LdadDataType type; + @XmlElement(name="v") + //@XmlList + public List values = new ArrayList(); + +} diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadTimeZoneMap.txt b/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadTimeZoneMap.txt new file mode 100644 index 0000000000..b068727c91 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadTimeZoneMap.txt @@ -0,0 +1,36 @@ +## +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------- -------- --------- -------------------------------------------- +# Mar 05, 2018 6851 randerso Initial creation +# +## + +## +# This is an absolute override file, indicating that a higher priority version +# of the file will completely replace a lower priority version of the file. +## + +## +# This file contains a mapping of time zone strings used by LDAD data +# to valid time zones recognized by java.time.ZoneId +## +EST GMT-0500 +EST5 GMT-0500 +CST GMT-0600 +CST6 GMT-0600 +MST GMT-0700 +MST7 GMT-0700 +PST GMT-0800 +PST8 GMT-0800 +AKST GMT-0900 +AKST9 GMT-0900 +AST10 GMT-1000 +HST GMT-1000 +HST10 GMT-1000 +HAST GMT-1000 +HAST10 GMT-1000 +ChST GMT+1000 +CHST GMT+1000 diff --git a/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadUnitsMap.txt b/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadUnitsMap.txt new file mode 100644 index 0000000000..eb0e1a64d4 --- /dev/null +++ b/edexOsgi/com.raytheon.edex.plugin.ldad/utility/common_static/base/ldad/ldadUnitsMap.txt @@ -0,0 +1,305 @@ +#$Id: ldatUnitsMap.txt 2009-09-28 slav Exp $ +## +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------- -------- --------- -------------------------------------------- +# Sep 28, 2009 slav Initial Creation. +# Mar 05, 2018 6851 randerso Added description of override behavior +# +## + +## +# This is an absolute override file, indicating that a higher priority version +# of the file will completely replace a lower priority version of the file. +## + +#==================================================================================== +# LDAD FIELD UNITS +#-------------------------------------------------- +# dataProvider # Typical data providers: CDoT, KDoT, UDFCD, etc. +# homeWFO +# observationTime # observationTime +# SurfaceObsLocation location # latitude, longitude, elevation, stationId "RALC2" +# providerId # "110" "FA6026DA" Data Provider station Id +# stationName # "Ralston_Res" "BEN CREEK AIRSTRIP" ????????????????? +# handbook5Id # "" ???????????????? +# stationType # "STO" "RAWS" ???????????? +# reportTime # 1.247436157E9 time data was processed by the provider +# receivedTime # time data was received - seconds since 1-1-1970 +# numericWMOid # numeric WMO identification +# dataPlatformType # short -32767 moving (e.g. floating buoy or ship) +# tempChangeTime # time of temperature last change - seconds since 1970-1-1 00:00:00.0 +# rhChangeTime # time of last relative humidity change +# stationPressChangeTime # time of last station press change +# pressChangeChar # long_name = "character of pressure change" +# windDirChangeTime # seconds since 1970-1-1 00:00:00.0 +# windSpeedChangeTime +# windGustChangeTime +# skyCover # char ref FMH-1 +# visibilityStatus +# totalCloudCover # tenths +# presWeather # present weather ref FMH-1 +# lowLevelCloudType # lowLevelCloudType:long_name = "low level cloud type" +# midLevelCloudType # midLevelCloudType:long_name = "middle level cloud type" +# highLevelCloudType # highLevelCloudType:long_name = "high level cloud type" +# maxTempRecordPeriod # maxTempRecordPeriod:long_name = "maximum temperature recording period" +# minTempRecordPeriod # minTempRecordPeriod:long_name = "minimum temperature recording period" +# precipType # precipType:long_name = "precipitation type" +# timeSinceLastPcp # seconds +# solarRadChangeTime # seconds since 1970-1-1 00:00:00.0 +# rawMessage +# cloudBaseHeight # cloudBaseHeight:long_name = "height of the lowest cloud layer" +# precipIntensity # precipIntensity:long_name = "precipitation intensity" + +#---------------------------------------------------- +# ldad_mesonet VarName stored units +#-------------------------------------------------- +latitude degree_angle +longitude degree_angle +elevation m +platformTrueDirection degree_angle +platformTrueSpeed m/s +wetBulbTemperature K +stationPressure Pa +pressChange3Hour Pa +windDirMin degree_angle +windDirMax degree_angle +skyLayerBase m +visibility m +maximumTemperature K +minimumTemperature K +precipAccum mm +solarRadiation W/m^2 +seaSurfaceTemp K +wavePeriod s +waveHeight m +temperature K +dewpoint K +relHumidity % +windDir degree_angle +windSpeed m/s +windGust m/s +pressure Pa +seaLevelPressure Pa +altimeter Pa +precipRate m/s +fuelTemperature K +fuelMoisture % +soilTemperature K +soilMoisture % + +#-------------------------------------------------- +# ldad_hydro VarName stored units +#-------------------------------------------------- +# voltageBattery volt +# waterConductance microS/cm +# waterOxygen mg/l +# waterPH pH +# riverReportChangeTime +belowSurface m +riverStage m +poolElevation m +tailwaterStage m +riverVelocity km/h +riverInflow m^3/s +riverFlow m^3/s +computedOutflow m^3/s +waterTemperature K +windSpeedPeak m/s +precip5min mm +precip1hr mm +precip3hr mm +precip6hr mm +precip12hr mm +precip18hr mm +precip24hr mm + +# ==================================================== +# UNITS OF THERMODYNAMIC TEMPERATURE +# +# KELVIN KELVIN +# CELSIUS KELVIN @ 273.15 +# RANKINE KELVIN/1.8 +# FAHRENHEIT RANKINE @ 459.67 + +# C CELSIUS # `C' means `coulomb' +Celsius Celsius +celsius Celsius +degree_centigrade Celsius +degC Celsius +degreeC Celsius +degree_C Celsius +degree_c Celsius +deg_C Celsius +deg_c Celsius +degK K +degreeK K +degree_K K +degree_k K +deg_K K +deg_k K +K K +degF degree_fahrenheit +degreeF degree_fahrenheit +degree_F degree_fahrenheit +degree_f degree_fahrenheit +deg_F degree_fahrenheit +deg_f degree_fahrenheit +F degree_fahrenheit +Fahrenheit degree_fahrenheit +fahrenheit degree_fahrenheit + +# R RANKINE # `R' means `roentgen' +degR degree_rankine +degreeR degree_rankine +degree_R degree_rankine +degree_r degree_rankine +deg_R degree_rankine +deg_r degree_rankine +Rankine degree_rankine +rankine degree_rankine +# ----------------------------------------- +# Relative humidity +% % +# ----------------------------------------- +# UNITS OF LENGTH +# +m m +meter m +meters m +metre m +metres m +mm mm + +# God help us! There's an international foot and a US survey foot and +# they're not the same! + +# International foot stuff: +international_inch in +international_foot ft +international_feet ft +international_yard yd +international_mile mi + +# Alias unspecified units to the international units: +inch in +foot ft +yard yd +mile mi + +# The following should hold regardless: +inches in +in in +ft ft +feet ft +yd yd +yards yd + +nmile nmi +nmi nmi + +# ----------------------------------------- +# UNITS OF TIME +# +day d +hour h +minute min +s s +sec s +common_year year +d day +min min +hr h +h h +week week +year year +yr year +a year +month month + +# ----------------------------------------- +# UNITS OF PLANE ANGLE +# +# rad rad +# circle rev +degree_angle degree_angle +angular_degree degree_angle + +# turn rev +degree degree_angle +degree_true degree_angle +arcdeg degree_angle +angular_minute degree_angle/60 +angular_second degree_angle/3600 + + +degree_north degree_angle +degreeN degree_angle +degree_N degree_angle +degrees_north degree_angle +degreesN degree_angle +degrees_N degree_angle + +degree_east degree_angle +degreeE degree_angle +degree_E degree_angle +degrees_east degree_angle +degreesE degree_angle +degrees_E degree_angle + +degree_west degree_angle*-1 +degreeW degree_angle*-1 +degree_W degree_angle*-1 +degrees_west degree_angle*-1 +degreesW degree_angle*-1 +degrees_W degree_angle*-1 + +degrees_true degree_angle +degreeT degree_angle +degree_T degree_angle +degreesT degree_angle +degrees_T degree_angle + +# ----------------------------------------- +# PRESSURE OR STRESS +# +bar bar +mB bar/1000 +standard_atmosphere atm +technical_atmosphere kgf/cm^2 +Pa Pa +inch_Hg inHg +inch_hg inHg +inHg inHg +in_Hg inHg +in_hg inHg +millimeter_Hg mmHg +mmHg mmHg +mm_Hg mmHg +mm_hg mmHg +torr mmHg +# at kgf/cm^2 +atmosphere atm +atm atm + +# ----------------------------------------- +# VELOCITY (INCLUDES SPEED) +# +# c m/s*299792458 +knot kn + +knot_international kn +international_knot kn +kt kn +m/s m/s +mph mph +kph km/h +mps m/s +iph in/h +mmph mm/h + +# ----------------------------------------- +# SOLAR RADIATION +W/meter2 W/m^2 diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.classpath b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.classpath new file mode 100644 index 0000000000..1fa3e6803d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.project b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.project new file mode 100644 index 0000000000..e4513cbdd1 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.common.dataplugin.ldad + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..79e7d3b670 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/META-INF/MANIFEST.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: LDAD Common +Bundle-SymbolicName: com.raytheon.uf.common.dataplugin.ldad +Bundle-Version: 1.18.0.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Export-Package: com.raytheon.uf.common.dataplugin.ldad +Require-Bundle: javax.persistence, + com.raytheon.uf.common.dataplugin, + com.raytheon.uf.common.pointdata +Import-Package: com.raytheon.uf.common.geospatial, + com.raytheon.uf.common.serialization.annotations diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/build.properties b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/build.properties new file mode 100644 index 0000000000..34d2e4d2da --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/src/com/raytheon/uf/common/dataplugin/ldad/LdadRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/src/com/raytheon/uf/common/dataplugin/ldad/LdadRecord.java new file mode 100644 index 0000000000..64945d5bf0 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldad/src/com/raytheon/uf/common/dataplugin/ldad/LdadRecord.java @@ -0,0 +1,126 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.ldad; + +import java.util.Date; + +import javax.persistence.MappedSuperclass; +import javax.persistence.SequenceGenerator; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.persist.PersistablePluginDataObject; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; + +/** + * Abstract base class for LDAD records + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- -----------------
+ * Mar 06, 2018  6851     randerso  Initial creation
+ * Apr 24, 2019  6140     tgurney   Remove Inheritance annotation
+ *                                  (Hibernate 5.4 fix)
+ *
+ *
+ * + * @author randerso + */ + +@MappedSuperclass +@SequenceGenerator(name = PluginDataObject.ID_GEN) +@DynamicSerialize +public abstract class LdadRecord extends PersistablePluginDataObject { + private static final long serialVersionUID = 1L; + + /* + * TODO: move common fields from MesonetLdadRecord and HydroLdadRecord up to + * LdadRecord. Unfortunately this changes the dataURI so would require + * additional changes. + */ + + /** + * Default Constructor + */ + public LdadRecord() { + super(); + } + + /** + * Constructor for DataURI construction through base class. This is used by + * the notification service. + * + * @param uri + * A data uri applicable to this class. + */ + public LdadRecord(String uri) { + super(uri); + } + + /** + * @param dataProvider + * the dataProvider to set + */ + public abstract void setDataProvider(String dataProvider); + + /** + * @param stationType + * the stationType to set + */ + public abstract void setStationType(String stationType); + + /** + * @param reportTime + * the reportTime to set + */ + public abstract void setReportTime(long reportTime); + + /** + * @return the observationTime + */ + public abstract Date getObservationTime(); + + /** + * @param observationTime + * the observationTime to set + */ + public abstract void setObservationTime(Date observationTime); + + /** + * @param location + * the location to set + */ + public abstract void setLocation(SurfaceObsLocation location); + + /** + * @param rawMessage + * the rawMessage to set + */ + public abstract void setRawMessage(String rawMessage); + + /** + * @return record as rawMessage + */ + public abstract String toMessage(); + +} diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.classpath b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.classpath new file mode 100644 index 0000000000..1fa3e6803d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.project b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.project new file mode 100644 index 0000000000..825bff20ee --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.common.dataplugin.ldadmesonet + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..83534dfaff --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/META-INF/MANIFEST.MF @@ -0,0 +1,16 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Ldadmesonet Plug-in +Bundle-SymbolicName: com.raytheon.uf.common.dataplugin.ldadmesonet +Bundle-Version: 1.18.0.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Import-Package: com.raytheon.uf.common.geospatial, + com.raytheon.uf.common.serialization.annotations, + org.locationtech.jts.geom +Export-Package: com.raytheon.uf.common.dataplugin.ldadmesonet +Require-Bundle: javax.persistence, + com.raytheon.uf.common.dataplugin, + com.raytheon.uf.common.pointdata, + com.raytheon.uf.common.dataaccess, + com.raytheon.uf.common.dataplugin.ldad diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/build.properties b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/build.properties new file mode 100644 index 0000000000..5791d48d5f --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + res/ diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/com.raytheon.uf.common.dataplugin.ldadmesonet.ecl b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/com.raytheon.uf.common.dataplugin.ldadmesonet.ecl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/res/spring/ldadmesonet-common-dataaccess.xml b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/res/spring/ldadmesonet-common-dataaccess.xml new file mode 100644 index 0000000000..69f17e7af8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/res/spring/ldadmesonet-common-dataaccess.xml @@ -0,0 +1,13 @@ + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/src/com/raytheon/uf/common/dataplugin/ldadmesonet/MesonetLdadRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/src/com/raytheon/uf/common/dataplugin/ldadmesonet/MesonetLdadRecord.java new file mode 100644 index 0000000000..bdcabf1590 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ldadmesonet/src/com/raytheon/uf/common/dataplugin/ldadmesonet/MesonetLdadRecord.java @@ -0,0 +1,1702 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.dataplugin.ldadmesonet; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Embedded; +import javax.persistence.Entity; +import javax.persistence.Index; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import javax.persistence.Transient; +import javax.persistence.UniqueConstraint; + +import org.locationtech.jts.geom.Geometry; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.annotations.DataURI; +import com.raytheon.uf.common.dataplugin.annotations.NullString; +import com.raytheon.uf.common.dataplugin.ldad.LdadRecord; +import com.raytheon.uf.common.dataplugin.persist.IPersistable; +import com.raytheon.uf.common.geospatial.ISpatialEnabled; +import com.raytheon.uf.common.pointdata.IPointData; +import com.raytheon.uf.common.pointdata.PointDataView; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Record implementation for ldadmesonet plugin. + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- --------------------------------------------
+ * Sep 04, 2009           vkorolev  Initial creation
+ * Apr 04, 2013  1846     bkowal    Added an index on refTime and forecastTime
+ * Apr 12, 2013  1857     bgonzale  Added SequenceGenerator annotation.
+ * May 07, 2013  1869     bsteffen  Remove dataURI column from PluginDataObject.
+ * May 15, 2013  1869     bsteffen  Remove DataURI column from ldadmesonet.
+ * Aug 30, 2013  2298     rjpeter   Make getPluginName abstract
+ * Jul 20, 2015  4360     rferrel   Named unique constraint Made reportType and
+ *                                  dataProvider not nullable
+ * Mar 06, 2018  6851     randerso  Changed to extend LdadRecord. Code cleanup.
+ * May 10, 2018  7288     randerso  Removed unused and unimplemented
+ *                                  setStationId() method
+ * Aug 08, 2022  8892     tjensen   Update indexes for Hibernate 5
+ *
+ *
+ * + * @author vkorolev + */ + +@Entity +@SequenceGenerator(initialValue = 1, name = PluginDataObject.ID_GEN, sequenceName = "ldadmesonetseq") +/* + * Both refTime and forecastTime are included in the refTimeIndex since + * forecastTime is unlikely to be used. + */ +@Table(name = "ldadmesonet", uniqueConstraints = { + @UniqueConstraint(name = "uk_ldadmesonet_datauri_fields", columnNames = { + "stationid", "reftime", "reporttype", "dataprovider", + "latitude", "longitude" }) }, indexes = { + @Index(name = "%TABLE%_refTimeIndex", columnList = "refTime, forecastTime"), + @Index(name = "%TABLE%_stationIndex", columnList = "stationId") }) + +@DynamicSerialize +public class MesonetLdadRecord extends LdadRecord + implements ISpatialEnabled, IPointData, IPersistable { + + private static final long serialVersionUID = 1L; + + private static final String OBS_TIME_FMT = "%1$tY/% + + + + + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/.project b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/.project new file mode 100644 index 0000000000..8104b96273 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.common.dataplugin.qc + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..23f50943b0 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/META-INF/MANIFEST.MF @@ -0,0 +1,15 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Qc Plug-in +Bundle-SymbolicName: com.raytheon.uf.common.dataplugin.qc +Bundle-Version: 1.18.0.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: com.raytheon.uf.common.dataplugin, + com.raytheon.uf.common.pointdata +Export-Package: com.raytheon.uf.common.dataplugin.qc +Import-Package: com.raytheon.uf.common.geospatial, + com.raytheon.uf.common.pointdata.spatial, + com.raytheon.uf.common.serialization.annotations, + com.raytheon.uf.common.status, + javax.persistence diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/build.properties b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/build.properties new file mode 100644 index 0000000000..34d2e4d2da --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/build.properties @@ -0,0 +1,4 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + . diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/com.raytheon.uf.common.dataplugin.qc.ecl b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/com.raytheon.uf.common.dataplugin.qc.ecl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/resources/com.raytheon.uf.common.dataplugin.qc.properties b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/resources/com.raytheon.uf.common.dataplugin.qc.properties new file mode 100644 index 0000000000..b9b97f0ee2 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/resources/com.raytheon.uf.common.dataplugin.qc.properties @@ -0,0 +1 @@ +HDF5_PATH=hdf5/QC \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.qc/src/com/raytheon/uf/common/dataplugin/qc/QCRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/src/com/raytheon/uf/common/dataplugin/qc/QCRecord.java new file mode 100644 index 0000000000..a7cf22dd42 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.qc/src/com/raytheon/uf/common/dataplugin/qc/QCRecord.java @@ -0,0 +1,197 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ + +package com.raytheon.uf.common.dataplugin.qc; + +import javax.persistence.Column; +import javax.persistence.Embedded; +import javax.persistence.Entity; +import javax.persistence.Index; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import javax.persistence.UniqueConstraint; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.annotations.DataURI; +import com.raytheon.uf.common.dataplugin.annotations.NullString; +import com.raytheon.uf.common.dataplugin.persist.PersistablePluginDataObject; +import com.raytheon.uf.common.geospatial.ISpatialEnabled; +import com.raytheon.uf.common.geospatial.ISpatialObject; +import com.raytheon.uf.common.pointdata.IPointData; +import com.raytheon.uf.common.pointdata.PointDataView; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; +import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; + +/** + * Record class for QC data + * + * 
+ *
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 07, 2009 3408       bphillip    Initial creation
+ * Apr 04, 2013 1846       bkowal      Added an index on refTime and
+ *                                     forecastTime
+ * Apr 12, 2013 1857       bgonzale    Added SequenceGenerator annotation.
+ * May 07, 2013 1869       bsteffen    Remove dataURI column from
+ *                                     PluginDataObject.
+ * May 16, 2013 1869       bsteffen    Remove DataURI column from qc.
+ * Aug 30, 2013 2298       rjpeter     Make getPluginName abstract
+ * Feb 27, 2014 2852       rferrel     Add getter/setter to FakePointDataView.
+ * Jul 21, 2015 4360       rferrel     Named unique constraint.
+ * Jan 04, 2018 6861       njensen     Removed unnecessary fields, use PointDataView
+ * Aug 08, 2022 8892       tjensen     Update indexes for Hibernate 5
+ *
+ *
+ *
+ * + * @author bphillip + */ +@Entity +@SequenceGenerator(initialValue = 1, name = PluginDataObject.ID_GEN, sequenceName = "qcseq") +/* + * Both refTime and forecastTime are included in the refTimeIndex since + * forecastTime is unlikely to be used. + */ +@Table(name = "qc", uniqueConstraints = { + @UniqueConstraint(name = "uk_qc_datauri_fields", columnNames = { + "stationid", "reftime", "qcType", "latitude", + "longitude" }) }, indexes = { + @Index(name = "%TABLE%_refTimeIndex", columnList = "refTime, forecastTime"), + @Index(name = "%TABLE%_stationIndex", columnList = "stationId") }) + +@DynamicSerialize +public class QCRecord extends PersistablePluginDataObject + implements ISpatialEnabled, IPointData { + + private static final long serialVersionUID = -8836262244188895665L; + + @Embedded + @DataURI(position = 2, embedded = true) + @DynamicSerializeElement + private SurfaceObsLocation location; + + @Column(nullable = false, length = 20) + @DataURI(position = 1) + @NullString + @DynamicSerializeElement + private String qcType; + + @Column(length = 15) + private String ncSet; + + private PointDataView pointDataView; + + public QCRecord() { + + } + + public QCRecord(String uri) { + super(uri); + } + + public String getStationId() { + return location.getStationId(); + } + + /** + * @return the location + */ + public SurfaceObsLocation getLocation() { + return location; + } + + /** + * @param location + * the location to set + */ + public void setLocation(SurfaceObsLocation location) { + this.location = location; + } + + public float getLatitude() { + return location.getLatitude().floatValue(); + } + + public float getLongitude() { + return location.getLongitude().floatValue(); + } + + public float getElevation() { + return location.getElevation(); + } + + public static long getSerialVersionUID() { + return serialVersionUID; + } + + /** + * @return the ncSet + */ + public String getNcSet() { + return ncSet; + } + + /** + * @param ncSet + * the ncSet to set + */ + public void setNcSet(String ncSet) { + this.ncSet = ncSet; + } + + @Override + public ISpatialObject getSpatialObject() { + return location; + } + + /** + * @return the qcType + */ + public String getQcType() { + return qcType; + } + + /** + * @param qcType + * the qcType to set + */ + public void setQcType(String qcType) { + this.qcType = qcType; + } + + @Override + public PointDataView getPointDataView() { + return pointDataView; + } + + @Override + public void setPointDataView(PointDataView pointDataView) { + this.pointDataView = pointDataView; + } + + @Override + public String getPluginName() { + return "qc"; + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.dataplugins.feature/feature.xml b/edexOsgi/com.raytheon.uf.edex.dataplugins.feature/feature.xml index c005f283a0..03a8efb725 100644 --- a/edexOsgi/com.raytheon.uf.edex.dataplugins.feature/feature.xml +++ b/edexOsgi/com.raytheon.uf.edex.dataplugins.feature/feature.xml @@ -44,7 +44,20 @@ install-size="0" version="0.0.0" unpack="false"/> + + + + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/.project b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/.project new file mode 100644 index 0000000000..8b8354367c --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.edex.plugin.ldadmesonet + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..54dbeb466f --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Ldadmesonet Plug-in +Bundle-SymbolicName: com.raytheon.uf.edex.plugin.ldadmesonet +Bundle-Version: 1.18.0.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: com.raytheon.uf.common.dataplugin.ldad, + com.raytheon.uf.common.dataplugin.ldadmesonet, + com.raytheon.edex.common, + com.raytheon.uf.edex.pointdata +Import-Package: com.raytheon.uf.common.status diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/build.properties b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/build.properties new file mode 100644 index 0000000000..20a5272307 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/build.properties @@ -0,0 +1,6 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + utility/,\ + .,\ + res/ diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/com.raytheon.uf.edex.plugin.ldadmesonet.ecl b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/com.raytheon.uf.edex.plugin.ldadmesonet.ecl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonet.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonet.xml new file mode 100644 index 0000000000..0b7cba9011 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonet.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonetdb.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonetdb.xml new file mode 100644 index 0000000000..3c9a8bcf06 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/pointdata/ldadmesonetdb.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-common.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-common.xml new file mode 100644 index 0000000000..c16506082e --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-common.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-ingest.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-ingest.xml new file mode 100644 index 0000000000..5e7c8a20f0 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/res/spring/ldadmesonet-ingest.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + java.lang.Throwable + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/LdadmesonetPointDataTransform.java b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/LdadmesonetPointDataTransform.java new file mode 100644 index 0000000000..1410fbd131 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/LdadmesonetPointDataTransform.java @@ -0,0 +1,557 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.plugin.ldadmesonet; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.ldadmesonet.MesonetLdadRecord; +import com.raytheon.uf.common.pointdata.PointDataContainer; +import com.raytheon.uf.common.pointdata.PointDataDescription; +import com.raytheon.uf.common.pointdata.PointDataView; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.edex.plugin.ldadmesonet.dao.LdadMesonetDao; + +/** + * Transform LDAD MESONET records into Point Data Model. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#     Engineer    Description
+ * ------------ ----------  ----------- --------------------------
+ * Oct 09, 2009 DR2814      vkorolev    Initial creation
+ * Aug 30, 2013 2298        rjpeter     Make getPluginName abstract
+ * Jul 23, 2014 3410        bclement    location changed to floats
+ * Aug 15, 2014 3530        bclement    moved from common to edex
+ * 
+ *
+ * + * @author vkorolev + * @version 1 + */ + +public class LdadmesonetPointDataTransform { + + private LdadMesonetDao dao; + + private PointDataDescription description; + + // ------------------Common params (From OBS + // plugin)---------------------------- + private static final String PRESS_CHANGE3_HOUR = "pressChange3Hour"; + + private static final String PRESS_CHANGE_CHAR = "pressChangeChar"; + + private static final String ALTIMETER = "altimeter"; + + private static final String WIND_GUST = "windGust"; + + private static final String WIND_SPEED = "windSpeed"; + + private static final String DEWPOINT = "dewpoint"; + + private static final String TEMPERATURE = "temperature"; + + private static final String PRES_WEATHER = "presWeather"; + + private static final String VISIBILITY = "visibility"; + + private static final String LONGITUDE = "longitude"; + + private static final String LATITUDE = "latitude"; + + private static final String ELEVATION = "elevation"; + + private static final String STATION_NAME = "stationName"; + + private static final String DATAURI = "dataURI"; + + // ------------------From LDAD mesonet netCDF------------------------ + private static final String STORAGE_TYPE = "storageType"; + + private static final String STATION_ID = "stationId"; + + private static final String DATA_PROVIDER = "dataProvider"; + + private static final String HOME_WFO = "homeWFO"; + + private static final String OBSERVATION_TIME = "observationTime"; + + private static final String PROVIDER_ID = "providerId"; + + private static final String HANDBOOK5_ID = "handbook5Id"; + + private static final String STATION_TYPE = "stationType"; + + private static final String REPORT_TIME = "reportTime"; + + private static final String RECEIVED_TIME = "receivedTime"; + + private static final String NUMERICAL_WMO_ID = "numericWMOid"; + + private static final String DATA_PLATFORM_TYPE = "dataPlatformType"; + + private static final String PLATFORM_TRUE_DIRECTION = "platformTrueDirection"; + + private static final String PLARFORM_TRUE_SPEED = "platformTrueSpeed"; + + private static final String TEMP_CHANGE_TIME = "tempChangeTime"; + + private static final String WET_BULB_TEMPERATURE = "wetBulbTemperature"; + + private static final String RH_CHANGE_TIME = "rhChangeTime"; + + private static final String STATION_PRESSURE = "stationPressure"; + + private static final String STATION_PRESS_CHANGE_TIME = "stationPressChangeTime"; + + private static final String WIND_DIR_CHANGE_TIME = "windDirChangeTime"; + + private static final String WIND_SPEED_CHANGE_TIME = "windSpeedChangeTime"; + + private static final String WIND_GUST_CHANGE_TIME = "windGustChangeTime"; + + private static final String WIND_DIR_MIN = "windDirMin"; + + private static final String WIND_DIR_MAX = "windDirMax"; + + private static final String VISIBILITY_STATUS = "visibilityStatus"; + + private static final String TOTAL_CLOUD_COVER = "totalCloudCover"; + + private static final String CLOUD_BASE_HEIGHT = "cloudBaseHeight"; + + private static final String LOW_LEVEL_CLOUD_TYPE = "lowLevelCloudType"; + + private static final String MID_LEVEL_CLOUD_TYPE = "midLevelCloudType"; + + private static final String HIGH_LEVEL_CLOUD_TYPE = "highLevelCloudType"; + + private static final String MAX_TEMP_RECORD_PERIOD = "maxTempRecordPeriod"; + + private static final String MAXIMUM_TEMPERATURE = "maximumTemperature"; + + private static final String MIN_TEMP_RECORD_PERIOD = "minTempRecordPeriod"; + + private static final String MINIMUM_TEMPERATURE = "minimumTemperature"; + + private static final String PRECIP_ACCUM = "precipAccum"; + + private static final String PRECIP_TYPE = "precipType"; + + private static final String PRECIP_INTENSITY = "precipIntensity"; + + private static final String TIME_SINCE_LAST_PCP = "timeSinceLastPcp"; + + private static final String SOLAR_RADIATION = "solarRadiation"; + + private static final String SOLAR_RAD_CHANGE_TIME = "solarRadChangeTime"; + + private static final String SEA_SURFACE_TEMP = "seaSurfaceTemp"; + + private static final String WAVE_PERIOD = "wavePeriod"; + + private static final String WAVE_HEIGHT = "waveHeight"; + + private static final String RAW_MESONET = "rawMessage"; + + private static final String REL_HUMIDITY = "relHumidity"; + + private static final String WIND_DIR = "windDir"; + + private static final String PRESSURE = "pressure"; + + private static final String SEA_LEVEL_PRESSURE = "seaLevelPressure"; + + private static final String PRECIP_RATE = "precipRate"; + + private static final String FUEL_TEMPERATURE = "fuelTemperature"; + + private static final String FUEL_MOISTURE = "fuelMoisture"; + + private static final String SOIL_TEMPERATURE = "soilTemperature"; + + private static final String SOIL_MOISTURE = "soilMoisture"; + + /** + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! It is important to + * keep this up to date or risk breaking backwards compatibility + * + */ + private static final String[] ALL_PARAMS = { PRESS_CHANGE3_HOUR, + PRESS_CHANGE_CHAR, ALTIMETER, WIND_GUST, WIND_SPEED, DEWPOINT, + TEMPERATURE, PRES_WEATHER, VISIBILITY, LONGITUDE, LATITUDE, + SEA_LEVEL_PRESSURE, STATION_NAME, DATAURI, STORAGE_TYPE, ELEVATION, + STATION_ID, DATA_PROVIDER, HOME_WFO, OBSERVATION_TIME, PROVIDER_ID, + HANDBOOK5_ID, STATION_TYPE, REPORT_TIME, RECEIVED_TIME, + NUMERICAL_WMO_ID, DATA_PLATFORM_TYPE, PLATFORM_TRUE_DIRECTION, + PLARFORM_TRUE_SPEED, TEMP_CHANGE_TIME, WET_BULB_TEMPERATURE, + RH_CHANGE_TIME, STATION_PRESSURE, STATION_PRESS_CHANGE_TIME, + WIND_DIR_CHANGE_TIME, WIND_SPEED_CHANGE_TIME, + WIND_GUST_CHANGE_TIME, WIND_DIR_MIN, WIND_DIR_MAX, + VISIBILITY_STATUS, TOTAL_CLOUD_COVER, CLOUD_BASE_HEIGHT, + LOW_LEVEL_CLOUD_TYPE, MID_LEVEL_CLOUD_TYPE, HIGH_LEVEL_CLOUD_TYPE, + MAX_TEMP_RECORD_PERIOD, MAXIMUM_TEMPERATURE, + MIN_TEMP_RECORD_PERIOD, MINIMUM_TEMPERATURE, PRECIP_ACCUM, + PRECIP_TYPE, PRECIP_INTENSITY, TIME_SINCE_LAST_PCP, + SOLAR_RADIATION, SOLAR_RAD_CHANGE_TIME, SEA_SURFACE_TEMP, + WAVE_PERIOD, WAVE_HEIGHT, RAW_MESONET, REL_HUMIDITY, WIND_DIR, + PRESSURE, SEA_LEVEL_PRESSURE, PRECIP_RATE, FUEL_TEMPERATURE, + FUEL_MOISTURE, SOIL_TEMPERATURE, SOIL_MOISTURE }; + + public static final String ALL_PARAMS_LIST; + static { + StringBuffer sb = new StringBuffer(); + boolean first = true; + for (String s : ALL_PARAMS) { + if (!first) { + sb.append(", "); + } else { + first = false; + } + sb.append(s); + } + ALL_PARAMS_LIST = sb.toString(); + } + + // public LdadmesonetPointDataTransform() throws JAXBException, + // PluginException { + // this.description = getDescription("ldadmesonet"); + // logger.info("=============PointDataDescription loaded=============="); + // this.dao = new LdadMesonetDao("ldadmesonet"); + // } + + public LdadmesonetPointDataTransform() { + try { + this.dao = new LdadMesonetDao("ldadmesonet"); + this.description = dao.getPointDataDescription(null); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + public PluginDataObject[] toPointData(PluginDataObject[] pdo) { + + if (pdo.length > 0) { + Map pointMap = new HashMap(); + + for (PluginDataObject p : pdo) { + if (!(p instanceof MesonetLdadRecord)) { + continue; + } + + File f = this.dao.getFullFilePath(p); + + PointDataContainer pdc = pointMap.get(f); + if (pdc == null) { + pdc = PointDataContainer.build(this.description); + pointMap.put(f, pdc); + } + + MesonetLdadRecord mesor = (MesonetLdadRecord) p; + PointDataView pdv = buildView(pdc, mesor); + mesor.setPointDataView(pdv); + } + } + return pdo; + } + + private PointDataView buildView(PointDataContainer container, + MesonetLdadRecord record) { + PointDataView pdv = container.append(); + + if (record.getRawMessage() != null) { + pdv.setString(RAW_MESONET, record.getRawMessage()); + } + if (record.getSeaLevelPressure() != null) { + pdv.setFloat(SEA_LEVEL_PRESSURE, record.getSeaLevelPressure()); + } + + if (record.getObservationTime() != null) { + pdv.setLong(OBSERVATION_TIME, record.getDataTime().getRefTime() + .getTime()); + } + if (record.getVisibility() != null) { + pdv.setFloat(VISIBILITY, record.getVisibility()); + } + if (record.getTemperature() != null) { + pdv.setFloat(TEMPERATURE, record.getTemperature()); + } + if (record.getDewpoint() != null) { + pdv.setFloat(DEWPOINT, record.getDewpoint()); + } + if (record.getWindSpeed() != null) { + pdv.setFloat(WIND_SPEED, record.getWindSpeed()); + } + if (record.getWindGust() != null) { + pdv.setFloat(WIND_GUST, record.getWindGust()); + } + if (record.getAltimeter() != null) { + pdv.setFloat(ALTIMETER, record.getAltimeter()); + } + if (record.getPressChangeChar() != null) { + pdv.setInt(PRESS_CHANGE_CHAR, record.getPressChangeChar() + .intValue()); + } + if (record.getPressChange3Hour() != null) { + pdv.setFloat(PRESS_CHANGE3_HOUR, record.getPressChange3Hour()); + } + + // --------------------------------------LDAD mesonet + // specific------------------------ + + if (record.getReportTime() != null) { + pdv.setLong(REPORT_TIME, record.getReportTime()); + } + if (record.getReceivedTime() != null) { + pdv.setLong(RECEIVED_TIME, record.getReceivedTime().longValue()); + } + if (record.getNumericWMOid() != null) { + pdv.setLong(NUMERICAL_WMO_ID, record.getNumericWMOid()); + } + if (record.getDataPlatformType() != null) { + pdv.setInt(DATA_PLATFORM_TYPE, record.getDataPlatformType() + .intValue()); + } + if (record.getPlatformTrueDirection() != null) { + pdv.setFloat(PLATFORM_TRUE_DIRECTION, + record.getPlatformTrueDirection()); + } + if (record.getPlatformTrueSpeed() != null) { + pdv.setFloat(PLARFORM_TRUE_SPEED, record.getPlatformTrueSpeed()); + } + if (record.getTempChangeTime() != null) { + pdv.setLong(TEMP_CHANGE_TIME, record.getTempChangeTime() + .longValue()); + } + if (record.getWetBulbTemperature() != null) { + pdv.setFloat(WET_BULB_TEMPERATURE, record.getWetBulbTemperature()); + } + if (record.getRhChangeTime() != null) { + pdv.setLong(RH_CHANGE_TIME, record.getRhChangeTime().longValue()); + } + if (record.getStationPressure() != null) { + pdv.setFloat(STATION_PRESSURE, record.getStationPressure()); + } + if (record.getStationPressChangeTime() != null) { + pdv.setLong(STATION_PRESS_CHANGE_TIME, record + .getStationPressChangeTime().longValue()); + } + if (record.getWindDirChangeTime() != null) { + pdv.setLong(WIND_DIR_CHANGE_TIME, record.getWindDirChangeTime() + .longValue()); + } + if (record.getWindSpeedChangeTime() != null) { + pdv.setLong(WIND_SPEED_CHANGE_TIME, record.getWindSpeedChangeTime() + .longValue()); + } + if (record.getWindGustChangeTime() != null) { + pdv.setLong(WIND_GUST_CHANGE_TIME, record.getWindGustChangeTime() + .longValue()); + } + if (record.getWindDirMin() != null) { + pdv.setFloat(WIND_DIR_MIN, record.getWindDirMin()); + } + if (record.getWindDirMax() != null) { + pdv.setFloat(WIND_DIR_MAX, record.getWindDirMax()); + } + if (record.getVisibilityStatus() != null) { + pdv.setString(VISIBILITY_STATUS, record.getVisibilityStatus()); + } + if (record.getTotalCloudCover() != null) { + pdv.setFloat(TOTAL_CLOUD_COVER, record.getTotalCloudCover()); + } + if (record.getCloudBaseHeight() != null) { + pdv.setInt(CLOUD_BASE_HEIGHT, record.getCloudBaseHeight() + .intValue()); + } + if (record.getLowLevelCloudType() != null) { + pdv.setInt(LOW_LEVEL_CLOUD_TYPE, record.getLowLevelCloudType() + .intValue()); + } + if (record.getMidLevelCloudType() != null) { + pdv.setInt(MID_LEVEL_CLOUD_TYPE, record.getMidLevelCloudType() + .intValue()); + } + if (record.getHighLevelCloudType() != null) { + pdv.setInt(HIGH_LEVEL_CLOUD_TYPE, record.getHighLevelCloudType() + .intValue()); + } + if (record.getMinTempRecordPeriod() != null) { + pdv.setInt(MAX_TEMP_RECORD_PERIOD, record.getMinTempRecordPeriod() + .intValue()); + } + if (record.getMaximumTemperature() != null) { + pdv.setFloat(MAXIMUM_TEMPERATURE, record.getMaximumTemperature()); + } + if (record.getMinTempRecordPeriod() != null) { + pdv.setInt(MIN_TEMP_RECORD_PERIOD, record.getMinTempRecordPeriod() + .intValue()); + } + if (record.getMinimumTemperature() != null) { + pdv.setFloat(MINIMUM_TEMPERATURE, record.getMinimumTemperature()); + } + if (record.getPrecipAccum() != null) { + pdv.setFloat(PRECIP_ACCUM, record.getPrecipAccum()); + } + if (record.getPrecipType() != null) { + pdv.setInt(PRECIP_TYPE, record.getPrecipType().intValue()); + } + if (record.getPrecipIntensity() != null) { + pdv.setInt(PRECIP_INTENSITY, record.getPrecipIntensity().intValue()); + } + if (record.getTimeSinceLastPcp() != null) { + pdv.setLong(TIME_SINCE_LAST_PCP, record.getTimeSinceLastPcp() + .longValue()); + } + if (record.getSolarRadiation() != null) { + pdv.setFloat(SOLAR_RADIATION, record.getSolarRadiation()); + } + if (record.getSolarRadChangeTime() != null) { + pdv.setLong(SOLAR_RAD_CHANGE_TIME, record.getSolarRadChangeTime() + .longValue()); + } + if (record.getSeaSurfaceTemp() != null) { + pdv.setFloat(SEA_SURFACE_TEMP, record.getSeaSurfaceTemp()); + } + if (record.getWavePeriod() != null) { + pdv.setFloat(WAVE_PERIOD, record.getWavePeriod()); + } + if (record.getWaveHeight() != null) { + pdv.setFloat(WAVE_HEIGHT, record.getWaveHeight()); + } + if (record.getRelHumidity() != null) { + pdv.setFloat(REL_HUMIDITY, record.getRelHumidity()); + } + if (record.getWindDir() != null) { + pdv.setFloat(WIND_DIR, record.getWindDir()); + } + if (record.getPressure() != null) { + pdv.setFloat(PRESSURE, record.getPressure()); + } + if (record.getSeaLevelPressure() != null) { + pdv.setFloat(SEA_LEVEL_PRESSURE, record.getSeaLevelPressure()); + } + if (record.getPrecipRate() != null) { + pdv.setFloat(PRECIP_RATE, record.getPrecipRate()); + } + if (record.getFuelTemperature() != null) { + pdv.setFloat(FUEL_TEMPERATURE, record.getFuelTemperature()); + } + if (record.getFuelMoisture() != null) { + pdv.setFloat(FUEL_MOISTURE, record.getFuelMoisture()); + } + if (record.getSoilTemperature() != null) { + pdv.setFloat(SOIL_TEMPERATURE, record.getSoilTemperature()); + } + if (record.getSoilMoisture() != null) { + pdv.setFloat(SOIL_MOISTURE, record.getSoilMoisture()); + } + return pdv; + } + + public static MesonetLdadRecord toMesonetLdadRecord(PointDataView pdv) { + MesonetLdadRecord mr = new MesonetLdadRecord(); + mr.setAltimeter(pdv.getNumber(ALTIMETER).floatValue()); + mr.setDewpoint(pdv.getNumber(DEWPOINT).floatValue()); + SurfaceObsLocation loc = new SurfaceObsLocation( + pdv.getString(STATION_ID)); + float lat = pdv.getNumber(LATITUDE).floatValue(); + float lon = pdv.getNumber(LONGITUDE).floatValue(); + loc.assignLocation(lat, lon); + loc.setElevation(pdv.getNumber(ELEVATION).intValue()); + mr.setLocation(loc); + mr.setReportType(pdv.getString(STORAGE_TYPE)); + mr.setProviderId(pdv.getString(PROVIDER_ID)); + mr.setPressChange3Hour(pdv.getNumber(PRESS_CHANGE3_HOUR).floatValue()); + mr.setPressChangeChar((short) pdv.getInt(PRESS_CHANGE_CHAR)); + mr.setSeaLevelPressure(pdv.getNumber(SEA_LEVEL_PRESSURE).floatValue()); + mr.setTemperature(pdv.getNumber(TEMPERATURE).floatValue()); + mr.setVisibility(pdv.getNumber(VISIBILITY).floatValue()); + mr.setWindDir(pdv.getFloat(WIND_DIR)); + mr.setWindGust(pdv.getFloat(WIND_GUST)); + mr.setWindSpeed(pdv.getFloat(WIND_SPEED)); + mr.setDataURI(pdv.getString(DATAURI)); + mr.setReceivedTime(pdv.getNumber(RECEIVED_TIME).doubleValue()); + // ---------------------------------------------------------------------------------- + mr.setTempChangeTime(pdv.getNumber(TEMP_CHANGE_TIME).doubleValue()); + mr.setWetBulbTemperature(pdv.getFloat(WET_BULB_TEMPERATURE)); + mr.setRhChangeTime((Double) pdv.getNumber(RH_CHANGE_TIME)); + mr.setStationPressure(pdv.getFloat(STATION_PRESSURE)); + mr.setStationPressChangeTime((Double) pdv + .getNumber(STATION_PRESS_CHANGE_TIME)); + mr.setWindDirChangeTime((Double) pdv.getNumber(WIND_DIR_CHANGE_TIME)); + mr.setWindSpeedChangeTime((Double) pdv + .getNumber(WIND_SPEED_CHANGE_TIME)); + mr.setWindGustChangeTime((Double) pdv.getNumber(WIND_GUST_CHANGE_TIME)); + mr.setWindDirMin(pdv.getFloat(WIND_DIR_MIN)); + mr.setWindDirMax(pdv.getFloat(WIND_DIR_MAX)); + mr.setVisibilityStatus(pdv.getString(VISIBILITY_STATUS)); + mr.setTotalCloudCover(pdv.getFloat(TOTAL_CLOUD_COVER)); + mr.setCloudBaseHeight((Short) pdv.getNumber(CLOUD_BASE_HEIGHT)); + mr.setLowLevelCloudType((Short) pdv.getNumber(LOW_LEVEL_CLOUD_TYPE)); + mr.setMidLevelCloudType((Short) pdv.getNumber(MID_LEVEL_CLOUD_TYPE)); + mr.setHighLevelCloudType((Short) pdv.getNumber(HIGH_LEVEL_CLOUD_TYPE)); + mr.setMaxTempRecordPeriod((Short) pdv.getNumber(MAX_TEMP_RECORD_PERIOD)); + mr.setMaximumTemperature(pdv.getFloat(MAXIMUM_TEMPERATURE)); + mr.setMinTempRecordPeriod((Short) pdv.getNumber(MIN_TEMP_RECORD_PERIOD)); + mr.setMaximumTemperature(pdv.getFloat(MINIMUM_TEMPERATURE)); + mr.setPrecipAccum(pdv.getFloat(PRECIP_ACCUM)); + mr.setPrecipType((Short) pdv.getNumber(PRECIP_TYPE)); + mr.setPrecipIntensity((Short) pdv.getNumber(PRECIP_INTENSITY)); + mr.setTimeSinceLastPcp((Double) pdv.getNumber(TIME_SINCE_LAST_PCP)); + mr.setSolarRadiation(pdv.getFloat(SOLAR_RADIATION)); + mr.setSolarRadChangeTime((Double) pdv.getNumber(SOLAR_RAD_CHANGE_TIME)); + mr.setSeaSurfaceTemp(pdv.getFloat(SEA_SURFACE_TEMP)); + mr.setWavePeriod(pdv.getFloat(WAVE_PERIOD)); + mr.setWaveHeight(pdv.getFloat(WAVE_HEIGHT)); + mr.setRawMessage(pdv.getString(RAW_MESONET)); + mr.setRelHumidity(pdv.getFloat(REL_HUMIDITY)); + mr.setWindDir(pdv.getFloat(WIND_DIR)); + mr.setPressure(pdv.getFloat(PRESSURE)); + mr.setSeaLevelPressure(pdv.getFloat(SEA_LEVEL_PRESSURE)); + mr.setPrecipRate(pdv.getFloat(PRECIP_RATE)); + mr.setFuelTemperature(pdv.getFloat(FUEL_TEMPERATURE)); + mr.setFuelMoisture(pdv.getFloat(FUEL_MOISTURE)); + mr.setSoilTemperature(pdv.getFloat(SOIL_TEMPERATURE)); + mr.setSoilMoisture(pdv.getFloat(SOIL_MOISTURE)); + return mr; + } + + public static MesonetLdadRecord[] toMesonetLdadRecords( + PointDataContainer container) { + List records = new ArrayList(); + container.setCurrentSz(container.getAllocatedSz()); + for (int i = 0; i < container.getCurrentSz(); i++) { + PointDataView pdv = container.readRandom(i); + records.add(toMesonetLdadRecord(pdv)); + } + return records.toArray(new MesonetLdadRecord[records.size()]); + + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/dao/LdadMesonetDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/dao/LdadMesonetDao.java new file mode 100644 index 0000000000..b7fa03e29b --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/src/com/raytheon/uf/edex/plugin/ldadmesonet/dao/LdadMesonetDao.java @@ -0,0 +1,163 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.plugin.ldadmesonet.dao; + +import java.util.ArrayList; +import java.util.List; + +import com.raytheon.uf.common.dataplugin.PluginException; +import com.raytheon.uf.common.dataplugin.ldadmesonet.MesonetLdadRecord; +import com.raytheon.uf.common.dataquery.db.QueryParam; +import com.raytheon.uf.common.pointdata.spatial.ObStation; +import com.raytheon.uf.edex.database.DataAccessLayerException; +import com.raytheon.uf.edex.database.query.DatabaseQuery; +import com.raytheon.uf.edex.pointdata.PointDataPluginDao; +import com.raytheon.uf.edex.pointdata.spatial.ObStationDao; + +/** + * Data access object for accessing LDAD MESONET records in the database. + * + * 
+ *
+ * SOFTWARE HISTORY
+ *
+ * Date         Ticket#     Engineer    Description
+ * ------------ ----------  ----------- --------------------------
+ * 9/04/09                  vkorolev    Initial creation
+ * 10/09/09     DR2814      vkorolev    Refactor to Point Data Model
+ * Feb 27, 2013 1638        mschenke    Moved ObStationDao to edex pointdata plugin
+ * Aug 15, 2014 3530        bclement    moved from common to edex
+ * Feb 16, 2022 8608        mapeters    Remove populateDataStore override that matched super
+ *
+ *
+ * + * @author vkorolev + */ +public class LdadMesonetDao extends PointDataPluginDao { + + /** The station dao */ + private ObStationDao obDao = new ObStationDao(); + + public LdadMesonetDao(String pluginName) throws PluginException { + super(pluginName); + } + + public LdadMesonetDao() throws PluginException { + this("ldadmesonet"); + } + + public List queryBySpatialBox(double upperLeftLat, double upperLeftLon, + double lowerRightLat, double lowerRightLon) + throws DataAccessLayerException { + + List stationList = obDao.queryBySpatialBox(upperLeftLat, + upperLeftLon, lowerRightLat, lowerRightLon); + + List stationNames = new ArrayList<>(); + for (ObStation station : stationList) { + stationNames.add(station.getIdentifier()); + } + stationList.clear(); + + DatabaseQuery query = new DatabaseQuery(MesonetLdadRecord.class); + query.addQueryParam("location.stationId", stationNames, + QueryParam.QueryOperand.IN); + return queryByCriteria(query); + } + + public List queryByState(String state, Integer count) + throws DataAccessLayerException { + + List results = obDao.queryByState(state); + + List icaos = new ArrayList<>(); + for (ObStation station : results) { + icaos.add(station.getIdentifier()); + } + + DatabaseQuery query = new DatabaseQuery(MesonetLdadRecord.class, count); + query.addQueryParam("location.stationId", icaos, + QueryParam.QueryOperand.IN); + return queryByCriteria(query); + } + + /** + * Retrieves an ldadMesonet report using the datauri . + * + * @param dataURI + * The dataURI to match against. + * @return The report record if it exists. + */ + public MesonetLdadRecord queryByDataURI(String dataURI) { + MesonetLdadRecord report = null; + List obs = null; + try { + obs = queryBySingleCriteria("dataURI", dataURI); + } catch (DataAccessLayerException e) { + logger.error("Error querying LDAD Mesonet data by URI: " + dataURI, + e); + } + if (obs != null && !obs.isEmpty()) { + report = (MesonetLdadRecord) obs.get(0); + } + return report; + } + + /** + * Queries for to determine if a given data uri exists on the redbook table. + * + * @param dataUri + * The DataURI to find. + * @return An array of objects. If not null, there should only be a single + * element. + */ + public Object[] queryDataUriColumn(final String dataUri) { + + String sql = "select datauri from awips.ldadmesonet where datauri='" + + dataUri + "';"; + + Object[] results = executeSQLQuery(sql); + + return results; + } + + public ObStationDao getObDao() { + return obDao; + } + + public void setObDao(ObStationDao obDao) { + this.obDao = obDao; + } + + @Override + public String[] getKeysRequiredForFileName() { + return new String[] { "dataTime.refTime" }; + } + + @Override + public String getPointDataFileName(MesonetLdadRecord p) { + return "ldadmesonet.h5"; + } + + @Override + public MesonetLdadRecord newObject() { + return new MesonetLdadRecord(); + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/distribution/ldadmesonet.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/distribution/ldadmesonet.xml new file mode 100644 index 0000000000..ad3b39d7a3 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/distribution/ldadmesonet.xml @@ -0,0 +1,29 @@ + + + + + + LDAD\.mesonet\.* + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/purge/ldadmesonetPurgeRules.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/purge/ldadmesonetPurgeRules.xml new file mode 100644 index 0000000000..f305712e99 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ldadmesonet/utility/common_static/base/purge/ldadmesonetPurgeRules.xml @@ -0,0 +1,36 @@ + + + + + + + 24 + =00-01:00:00 + 00-01:00:00 + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/.classpath b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.classpath new file mode 100644 index 0000000000..1fa3e6803d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/.project b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.project new file mode 100644 index 0000000000..37b171dac2 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.project @@ -0,0 +1,34 @@ + + + com.raytheon.uf.edex.plugin.qc + + + + + + org.python.pydev.PyDevBuilder + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + org.python.pydev.pythonNature + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/.pydevproject b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.pydevproject new file mode 100644 index 0000000000..faf12629d9 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/.pydevproject @@ -0,0 +1,7 @@ + + + + +python 2.5 +Default + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.plugin.qc/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..76044450d6 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/META-INF/MANIFEST.MF @@ -0,0 +1,13 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Qc Plug-in +Bundle-SymbolicName: com.raytheon.uf.edex.plugin.qc +Bundle-Version: 1.18.0.qualifier +Bundle-Vendor: RAYTHEON +Bundle-RequiredExecutionEnvironment: JavaSE-1.8 +Require-Bundle: com.raytheon.edex.common, + com.raytheon.uf.edex.pointdata, + com.raytheon.uf.common.dataplugin.qc, + com.raytheon.uf.common.localization, + ucar.nc2 +Import-Package: com.raytheon.uf.common.status diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/build.properties b/edexOsgi/com.raytheon.uf.edex.plugin.qc/build.properties new file mode 100644 index 0000000000..9d91e04a59 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/build.properties @@ -0,0 +1,7 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + utility/,\ + .,\ + res/,\ + resources/ diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/com.raytheon.uf.edex.plugin.qc.ecl b/edexOsgi/com.raytheon.uf.edex.plugin.qc/com.raytheon.uf.edex.plugin.qc.ecl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/ldadmesonet.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/ldadmesonet.xml new file mode 100644 index 0000000000..98dabd8ea9 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/ldadmesonet.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/msas.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/msas.xml new file mode 100644 index 0000000000..86a22c7105 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/pdd/msas.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/qcdb.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/qcdb.xml new file mode 100644 index 0000000000..b7e7f6b5ce --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/pointdata/qcdb.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-common.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-common.xml new file mode 100644 index 0000000000..3e0f421958 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-common.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-ingest.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-ingest.xml new file mode 100644 index 0000000000..56ae4706ae --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/res/spring/qc-ingest.xml @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + qc + + + + + + + java.lang.Throwable + + + + + + + + + + + + + + + + + + + + + + + java.lang.Throwable + + + + + + + + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/resources/qc.properties b/edexOsgi/com.raytheon.uf.edex.plugin.qc/resources/qc.properties new file mode 100644 index 0000000000..2d0bea424f --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/resources/qc.properties @@ -0,0 +1,2 @@ +# period to scan the QC file drop point for new QC data +qc.cron=0+2,7,12,17,22,27,32,37,42,47,52,57+*+*+*+? \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/QCScanner.java b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/QCScanner.java new file mode 100644 index 0000000000..4cde543c2d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/QCScanner.java @@ -0,0 +1,370 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ + +package com.raytheon.uf.edex.plugin.qc; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.qc.QCRecord; +import com.raytheon.uf.common.pointdata.PointDataContainer; +import com.raytheon.uf.common.pointdata.PointDataDescription; +import com.raytheon.uf.common.pointdata.PointDataView; +import com.raytheon.uf.common.pointdata.spatial.SurfaceObsLocation; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.time.DataTime; +import com.raytheon.uf.edex.database.plugin.PluginFactory; +import com.raytheon.uf.edex.plugin.qc.dao.QCDao; +import com.raytheon.uf.edex.plugin.qc.internal.QCPaths; + +import ucar.ma2.Array; +import ucar.ma2.ArrayChar; +import ucar.ma2.DataType; +import ucar.ma2.Section; +import ucar.nc2.Attribute; +import ucar.nc2.NetcdfFile; +import ucar.nc2.Variable; + +/** + * Scans NetCDF files generated by A1 legacy applications and generates QCRecord + * instances that refer to their records. The scan should only occur on one JVM + * in a cluster while the decode can occur on multiple JVMs in a cluster. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Dec 03, 2010 4775       D. Friedman Initial version
+ * Mar 07, 2013 15842      D. Friedman Use Java NetCDF library instead of
+ *                                     pupynere
+ * May 16, 2013 1869       bsteffen    Remove DataURI column from qc.
+ * Aug 30, 2013 2298       rjpeter     Make getPluginName abstract
+ * Feb 20, 2014 DR 17098   D. Friedman Filter out invalid lat/lon values.
+ * Jul 23, 2014 3410       bclement    location changed to floats
+ * Jan 03, 2018 6861       njensen     Split on files, decode and store results
+ *                                      as PointDataView on PluginDataObject[]
+ * 
+ *
+ * + * @author dfriedma + */ +public class QCScanner { + + private static final IUFStatusHandler statusHandler = UFStatus + .getHandler(QCScanner.class); + + private static final Pattern FILE_PATTERN = Pattern.compile("^\\d+_\\d+$"); + + private Integer maxRecordsInChunk; + + /** + * Map of qcType to filename to last mod time. Used to ensure we don't scan + * a file again if it hasn't changed. + */ + private Map> fileLastScannedMap = new HashMap<>(); + + /** + * @return the maxRecordsInChunk + */ + public Integer getMaxRecordsInChunk() { + return maxRecordsInChunk; + } + + /** + * @param maxRecordsInChunk + * the maxRecordsInChunk to set + */ + public void setMaxRecordsInChunk(Integer maxRecordsInChunk) { + this.maxRecordsInChunk = maxRecordsInChunk; + } + + /** + * Information necessary to start a decode. Serialized after the split to + * take advantage of clustering the decodes. + */ + public static class QCFile implements Serializable { + + private static final long serialVersionUID = 1L; + + protected String qcType; + + protected File file; + + protected QCFile(String qcType, File file) { + this.qcType = qcType; + this.file = file; + } + } + + public class QCDirectoryScanner { + private final String qcType; + + private final File directory; + + public QCDirectoryScanner(String qcType, File directory) { + this.qcType = qcType; + this.directory = directory; + } + + /** + * Scan all files in the directory that match the pattern and have been + * modified since we last scanned them. If we did not previously scan + * them, then scan those too. + * + * @return the list of QCFiles that match the pattern and have been + * modified since we last scanned them + */ + public List getFiles() { + List results = new ArrayList<>(); + Map updatedScanMap = new HashMap<>(); + Map innerMap = fileLastScannedMap.get(qcType); + if (innerMap == null) { + innerMap = new HashMap<>(); + } + + for (String fn : directory.list()) { + if (FILE_PATTERN.matcher(fn).matches()) { + try { + File f = new File(directory, fn); + String path = f.getPath(); + Long lastScanned = innerMap.get(path); + if (lastScanned == null + || f.lastModified() > lastScanned) { + QCFile qcf = new QCFile(qcType, f); + results.add(qcf); + lastScanned = f.lastModified(); + } + updatedScanMap.put(path, lastScanned); + } catch (Exception e) { + statusHandler + .error(String.format("error reading %s/%s: %s", + directory, fn, e.getMessage()), e); + } + } + } + + fileLastScannedMap.put(qcType, updatedScanMap); + return results; + } + } + + /** + * Gets an iterator of QC files to process/decode. + * + * @return an iterator of QC files + * @throws FileNotFoundException + */ + public Iterator split() throws FileNotFoundException { + List files = new ArrayList<>(); + Map paths = QCPaths.getPaths(); + for (Map.Entry entry : paths.entrySet()) { + String type = entry.getKey(); + File directory = entry.getValue(); + QCDirectoryScanner scanner = new QCDirectoryScanner(type, + directory); + files.addAll(scanner.getFiles()); + } + + return files.iterator(); + } + + /** + * Scans or "decodes" the QC netcdf file into QCRecords + * + * @param qcFile + * the QC file to scan for records + * @return the PluginDataObjects that were "decoded" + * @throws Exception + */ + public PluginDataObject[] scanFile(QCFile qcFile) throws Exception { + try (NetcdfFile nc = NetcdfFile.open(qcFile.file.getPath())) { + int index = 0; + int totalRecordCount = nc.getUnlimitedDimension().getLength(); + int batchSize = Math.min(getMaxRecordsInChunk(), totalRecordCount); + + String[] idVariablesNames = nc.findGlobalAttribute("idVariables") + .getStringValue().split(","); + String[] timeVariableNames = nc.findGlobalAttribute("timeVariables") + .getStringValue().split(","); + Variable[] idVariables = new Variable[idVariablesNames.length]; + for (int i = 0; i < idVariables.length; ++i) { + idVariables[i] = nc.findVariable(idVariablesNames[i]); + } + Variable vObsTime = nc.findVariable(timeVariableNames[0]); + double vObsTimeFillValue = vObsTime.findAttribute("_FillValue") + .getNumericValue().doubleValue(); + Double vObsTimeMissingValue = null; + Attribute a = vObsTime.findAttribute("missing_value"); + if (a != null) { + vObsTimeMissingValue = a.getNumericValue().doubleValue(); + } + Variable vLat = nc.findVariable("latitude"); + Variable vLon = nc.findVariable("longitude"); + Variable vElev = nc.findVariable("elevation"); + + /* + * There's so many variables in the file that we don't need so we + * pick the ones we do need from the point data description. + */ + QCDao qcDao = (QCDao) PluginFactory.getInstance() + .getPluginDao("qc"); + Map map = new HashMap<>(); + map.put("qcType", qcFile.qcType); + PointDataDescription pdd = qcDao.getPointDataDescription(map); + PointDataContainer pdc = PointDataContainer.build(pdd); + Map variableMap = new HashMap<>(); + for (String name : pdd.getParameterNames()) { + /* + * do not include id variables, we will read those separately + * below because they are 2-dimensional chars (aka Strings) + */ + boolean foundId = false; + for (String idName : idVariablesNames) { + if (idName.equals(name)) { + foundId = true; + break; + } + } + if (!foundId) { + Variable v = nc.findVariable(name); + if (v != null) { + variableMap.put(name, v); + } + } + } + + Map recordMap = new HashMap<>(); + while (index < totalRecordCount) { + batchSize = Math.min(batchSize, totalRecordCount - index); + int[] ofs = new int[] { index }; + int[] len = new int[] { batchSize }; + Array dObsTime = vObsTime.read(ofs, len); + Array dLat = vLat.read(ofs, len); + Array dLon = vLon.read(ofs, len); + Array dElev = vElev.read(ofs, len); + + // get IDs which will become the unique station name + Section sec = new Section(); + sec.appendRange(index, (index + batchSize) - 1); + sec.appendRange(); + ArrayChar[] dIDs = new ArrayChar[idVariables.length]; + for (int i = 0; i < dIDs.length; ++i) { + dIDs[i] = (ArrayChar) idVariables[i].read(sec); + } + + Map nameToArrayMap = new HashMap<>(); + for (Variable v : variableMap.values()) { + Array arr = v.read(ofs, len); + nameToArrayMap.put(v.getName(), arr); + } + + int ri = 0; + while (ri < batchSize) { + PointDataView pdv = pdc.append(); + QCRecord pdo = new QCRecord(); + pdo.setOverwriteAllowed(true); + double obsTime = dObsTime.getDouble(ri); + float lat = dLat.getFloat(ri); + float lon = dLon.getFloat(ri); + if ((obsTime != vObsTimeFillValue) + && ((vObsTimeMissingValue == null) + || (vObsTimeMissingValue != obsTime)) + && Math.abs(lon) <= 180 && Math.abs(lat) <= 90) { + pdo.setDataTime(new DataTime( + new Date((long) (obsTime * 1000)))); + SurfaceObsLocation loc = new SurfaceObsLocation(); + loc.assignLocation(lat, lon); + loc.setElevation(dElev.getInt(ri)); + + /* + * Set up a unique stationId while also setting the id + * variable names with point data values + */ + StringBuilder stationId = new StringBuilder(); + for (ArrayChar idArray : dIDs) { + stationId.append(idArray.getString(ri)); + } + loc.setStationId(stationId.toString()); + for (int i = 0; i < idVariablesNames.length; i++) { + pdv.setString(idVariablesNames[i], + dIDs[i].getString(ri)); + } + + pdo.setLocation(loc); + pdo.setNcSet(qcFile.file.getName()); + pdo.setQcType(qcFile.qcType); + + for (String name : nameToArrayMap.keySet()) { + Array arr = nameToArrayMap.get(name); + DataType dtype = arr.getDataType(); + switch (dtype) { + case CHAR: + pdv.setString(name, + String.valueOf(arr.getChar(ri))); + break; + case INT: + case SHORT: + pdv.setInt(name, arr.getInt(ri)); + break; + case LONG: + pdv.setLong(name, arr.getLong(ri)); + break; + case FLOAT: + case DOUBLE: + pdv.setFloat(name, arr.getFloat(ri)); + break; + default: + throw new IllegalArgumentException( + "QCScanner does not support reading values of type " + + dtype); + } + + } + pdo.setPointDataView(pdv); + recordMap.put(pdo.getDataURI(), pdo); + } + ++index; + ++ri; + + } + + } // end of outer while + + Collection results = recordMap.values(); + int size = results.size(); + statusHandler.info("Decoded " + size + " QC records from " + + qcFile.file.getPath() + ", now storing"); + return results.toArray(new PluginDataObject[size]); + } + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/dao/QCDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/dao/QCDao.java new file mode 100644 index 0000000000..a48d107df8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/dao/QCDao.java @@ -0,0 +1,92 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ + +package com.raytheon.uf.edex.plugin.qc.dao; + +import java.util.HashMap; +import java.util.Map; + +import com.raytheon.uf.common.dataplugin.PluginException; +import com.raytheon.uf.common.dataplugin.qc.QCRecord; +import com.raytheon.uf.common.pointdata.PointDataDescription; +import com.raytheon.uf.edex.plugin.qc.internal.QCPaths; +import com.raytheon.uf.edex.pointdata.PointDataPluginDao; + +/** + * Data access object for retrieving QC mesonet data. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * 12/04/2009   3408       bphillip    Initial creation
+ * Aug 15, 2014 3530       bclement    removed commons logging
+ * Apr 16, 2015 4259       njensen     Removed unreachable catch
+ * Jan 04, 2018 6861       njensen     Removed python usage, removed method overrides
+ *                                      so it mostly resembles a normal pointdata plugin
+ * 
+ *
+ * + * @author bphillip + */ +public class QCDao extends PointDataPluginDao { + + /** Map of plugin names to point data descriptions */ + private static Map pdds = new HashMap<>(); + + static { + pdds = QCPaths.getPointDataDescriptions(); + } + + /** + * Constructs a new QC data access object + * + * @param pluginName + * "qc" + * @throws PluginException + * If errors occur while constructing the data access object + */ + public QCDao(String pluginName) throws PluginException { + super(pluginName); + } + + @Override + public String[] getKeysRequiredForFileName() { + return new String[] { "qcType", "dataTime.refTime" }; + } + + @Override + public String getPointDataFileName(QCRecord p) { + return "qc.h5"; + } + + @Override + public QCRecord newObject() { + return new QCRecord(); + } + + @Override + public PointDataDescription getPointDataDescription( + Map obj) { + return pdds.get(obj.get("qcType").toString()); + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/internal/QCPaths.java b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/internal/QCPaths.java new file mode 100644 index 0000000000..45bd9e92d8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/src/com/raytheon/uf/edex/plugin/qc/internal/QCPaths.java @@ -0,0 +1,123 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ + +package com.raytheon.uf.edex.plugin.qc.internal; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import com.raytheon.uf.common.pointdata.PointDataDescription; +import com.raytheon.uf.common.serialization.SerializationException; +import com.raytheon.uf.edex.core.EDEXUtil; + +/** + * This class should only be used by the QC plug-ins. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date          Ticket#  Engineer  Description
+ * ------------- -------- --------- --------------------------------------------
+ * ???           ???      ???       Initial creation
+ * Aug 15, 2014  3530     bclement  moved from common to edex
+ * Jul 15, 2016  5744     mapeters  Use common_static instead of edex_static
+ *                                  in getPythonScriptPath()
+ * Jan 05, 2018  6861     njensen   Removed python support
+ *                                  Simplified getPointDataDescriptions()
+ * 
+ *
+ * + */ +public class QCPaths { + + /** The directory containing the raw QC mesonet data */ + private static final String QC_RAW_DIR; + + static { + QC_RAW_DIR = EDEXUtil.getEdexData() + File.separator + + System.getProperty("HDF5_PATH"); + } + + public static Map getPaths() throws FileNotFoundException { + Map paths = new HashMap<>(); + File baseDir = new File(QC_RAW_DIR); + if (baseDir.exists()) { + File[] files = baseDir.listFiles(); + if (files == null) { + throw new FileNotFoundException( + "Unable to read files in directory: " + baseDir); + } + for (File f : files) { + paths.put(f.getName(), f); + } + } else { + throw new FileNotFoundException( + "Unable to open directory: " + baseDir); + } + return paths; + } + + public static Map getPointDataDescriptions() { + Map pdds = new HashMap<>(); + String mesonetPath = "/res/pointdata/pdd/ldadmesonet.xml"; + String msasPath = "/res/pointdata/pdd/msas.xml"; + + try (InputStream is = QCPaths.class.getResourceAsStream(mesonetPath)) { + PointDataDescription desc; + try { + desc = PointDataDescription.fromStream(is); + pdds.put("ldadmesonet", desc); + } catch (SerializationException e) { + throw new RuntimeException( + "Error reading pointdata description from " + + mesonetPath, + e); + } + } catch (IOException e) { + // ignore + } + + try (InputStream is = QCPaths.class.getResourceAsStream(msasPath)) { + PointDataDescription desc; + try { + desc = PointDataDescription.fromStream(is); + pdds.put("msas", desc); + } catch (SerializationException e) { + throw new RuntimeException( + "Error reading pointdata description from " + msasPath, + e); + } + } catch (IOException e) { + // ignore + } + + return pdds; + } + + public static String getBaseDir() { + return QC_RAW_DIR; + } + +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.qc/utility/common_static/base/path/qcPathKeys.xml b/edexOsgi/com.raytheon.uf.edex.plugin.qc/utility/common_static/base/path/qcPathKeys.xml new file mode 100644 index 0000000000..eddffb6f5c --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.plugin.qc/utility/common_static/base/path/qcPathKeys.xml @@ -0,0 +1,32 @@ + + + + + + + qcType + 0 + +