xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1804 Remove empty data structures from radar hdf5.

Browse source 
Change-Id: Ide845d6a5b308f61dc070f3acb01b96ed9a9288d

Former-commit-id: 41372ac122 [formerly 1203b7cee4] [formerly ee7204c05c] [formerly 41372ac122 [formerly 1203b7cee4] [formerly ee7204c05c] [formerly 82499f839d [formerly ee7204c05c [formerly 31528bbb403ec0fd6ee792f4451ac209fb272dcd]]]]
Former-commit-id: 82499f839d
Former-commit-id: d67d0ec674 [formerly 1b363a547c] [formerly 048a16f2b52da3414fc5d576059d5543b43f69f9 [formerly 3e2a41d1d2]]
Former-commit-id: ce0f7dafa19ff3c41040e73684d15a758d74d293 [formerly 57d19b0d66]
Former-commit-id: a01b552f89
This commit is contained in:
Ben Steffensmeier 2013-03-19 16:59:51 -05:00
parent 0af3490317
commit ce8d190a49
4 changed files with 131 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cave/com.raytheon.viz.radar/src/com/raytheon/viz/radar/ui/xy/RadarXYResource.java
View file

@ -35,6 +35,7 @@ import com.raytheon.uf.common.dataplugin.radar.RadarRecord;
import com.raytheon.uf.common.dataplugin.radar.level3.Layer; import com.raytheon.uf.common.dataplugin.radar.level3.Layer;
import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVector; import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVector;
import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVectorPacket; import com.raytheon.uf.common.dataplugin.radar.level3.LinkedVectorPacket;
import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyBlock;
import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket; import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyPacket;
import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket; import com.raytheon.uf.common.dataplugin.radar.level3.TextSymbolPacket;
import com.raytheon.uf.common.dataplugin.radar.level3.UnlinkedVector; import com.raytheon.uf.common.dataplugin.radar.level3.UnlinkedVector;
@ -78,6 +79,8 @@ import com.vividsolutions.jts.geom.Coordinate;
* Mar 16, 2009 askripsk Initial creation * Mar 16, 2009 askripsk Initial creation
* Jul 26, 2010 #3723 bkowal Now implements the magnification * Jul 26, 2010 #3723 bkowal Now implements the magnification
* capability. * capability.
* Mar 19, 2013 1804 bsteffen Remove empty data structures from radar
* hdf5.
* *
* </pre> * </pre>
* *
@ -335,7 +338,9 @@ public class RadarXYResource extends RadarImageResource<RadarXYDescriptor> {
linkedLines.clear(); linkedLines.clear();
unlinkedLines.clear(); unlinkedLines.clear();
points.clear(); points.clear();
for (Layer currLayer : radarRecord.getSymbologyBlock().getLayers()) { SymbologyBlock sb = radarRecord.getSymbologyBlock();
if (sb != null) {
for (Layer currLayer : sb.getLayers()) {
for (SymbologyPacket currPacket : currLayer.getPackets()) { for (SymbologyPacket currPacket : currLayer.getPackets()) {
if (currPacket instanceof TextSymbolPacket) { if (currPacket instanceof TextSymbolPacket) {
TextSymbolPacket tsp = (TextSymbolPacket) currPacket; TextSymbolPacket tsp = (TextSymbolPacket) currPacket;
@ -358,6 +363,7 @@ public class RadarXYResource extends RadarImageResource<RadarXYDescriptor> {
} }
} }
} }
}
initPlotObjects = false; initPlotObjects = false;
} }

22
edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/RadarDecoder.java
View file

@ -86,9 +86,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* 2/14/2007 139 Phillippe Initial check-in. Refactor of initial implementation. * 2/14/2007 139 Phillippe Initial check-in. Refactor of initial implementation.
* Dec 17, 2007 600 bphillip Added dao pool usage * Dec 17, 2007 600 bphillip Added dao pool usage
* Dec 03, 2010 2235 cjeanbap EDEXUtility.sendMessageAlertViz() signature changed. * Dec 03, 2010 2235 cjeanbap EDEXUtility.sendMessageAlertViz() signature changed.
* Mar 18, 2013 1804 bsteffen Remove AlphanumericValues from radar * Mar 19, 2013 1804 bsteffen Optimize decoder performance.
* HDF5.
* Mar 19, 2013 1804 bsteffen Cache db queries in radar decoder.
* *
* </pre> * </pre>
* *
@ -547,13 +545,12 @@ public class RadarDecoder extends AbstractDecoder {
*/ */
private void processSymbologyBlock(RadarRecord record, private void processSymbologyBlock(RadarRecord record,
SymbologyBlock symbologyBlock) { SymbologyBlock symbologyBlock) {
int errorCount = 0;
if (symbologyBlock == null) { if (symbologyBlock == null) {
return; return;
} }
int packetsKept = 0;
List<Layer> packetsInLyrs = new ArrayList<Layer>(); List<Layer> packetsInLyrs = new ArrayList<Layer>();
for (int layer = 0; layer < symbologyBlock.getNumLayers(); ++layer) { for (int layer = 0; layer < symbologyBlock.getNumLayers(); ++layer) {
Layer lyr = new Layer(); Layer lyr = new Layer();
@ -589,20 +586,19 @@ public class RadarDecoder extends AbstractDecoder {
} }
} }
} }
packetsKept += packets.size();
lyr.setPackets(packets.toArray(new SymbologyPacket[packets.size()])); lyr.setPackets(packets.toArray(new SymbologyPacket[packets.size()]));
packetsInLyrs.add(lyr); packetsInLyrs.add(lyr);
} }
// remove the radial and raster from the symb block // remove the radial and raster from the symb block, only keep it if
symbologyBlock.setLayers(packetsInLyrs.toArray(new Layer[packetsInLyrs // there are other packets.
.size()])); if (packetsKept > 0) {
symbologyBlock.setLayers(packetsInLyrs
.toArray(new Layer[packetsInLyrs.size()]));
record.setSymbologyBlock(symbologyBlock); record.setSymbologyBlock(symbologyBlock);
record.correlateSymbologyPackets(); record.correlateSymbologyPackets();
if (errorCount > 0) {
logger.error("Radar file contains " + errorCount
+ " unrecognized symbology packet types.");
} }
} }

32
edexOsgi/com.raytheon.edex.plugin.radar/src/com/raytheon/edex/plugin/radar/dao/RadarDao.java
View file

@ -28,8 +28,8 @@ package com.raytheon.edex.plugin.radar.dao;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 02/06/09 1990 bphillip Initial creation * 02/06/09 1990 bphillip Initial creation
* Mar 18, 2013 1804 bsteffen Remove AlphanumericValues from radar * Mar 18, 2013 1804 bsteffen Reduce useless data stored in radar hdf5
* HDF5. *
* </pre> * </pre>
* *
* @author bphillip * @author bphillip
@ -43,12 +43,15 @@ import java.util.Map;
import com.raytheon.uf.common.dataplugin.PluginDataObject; import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.dataplugin.PluginException; import com.raytheon.uf.common.dataplugin.PluginException;
import com.raytheon.uf.common.dataplugin.persist.IPersistable; import com.raytheon.uf.common.dataplugin.persist.IPersistable;
import com.raytheon.uf.common.dataplugin.radar.RadarDataKey;
import com.raytheon.uf.common.dataplugin.radar.RadarDataPoint;
import com.raytheon.uf.common.dataplugin.radar.RadarRecord; import com.raytheon.uf.common.dataplugin.radar.RadarRecord;
import com.raytheon.uf.common.dataplugin.radar.RadarStoredData; import com.raytheon.uf.common.dataplugin.radar.RadarStoredData;
import com.raytheon.uf.common.dataplugin.radar.level3.GSMBlock.GSMMessage; import com.raytheon.uf.common.dataplugin.radar.level3.GSMBlock.GSMMessage;
import com.raytheon.uf.common.dataplugin.radar.level3.GraphicBlock; import com.raytheon.uf.common.dataplugin.radar.level3.GraphicBlock;
import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyBlock; import com.raytheon.uf.common.dataplugin.radar.level3.SymbologyBlock;
import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants; import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants;
import com.raytheon.uf.common.dataplugin.radar.util.RadarConstants.MapValues;
import com.raytheon.uf.common.dataplugin.radar.util.RadarDataRetriever; import com.raytheon.uf.common.dataplugin.radar.util.RadarDataRetriever;
import com.raytheon.uf.common.datastorage.IDataStore; import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageProperties; import com.raytheon.uf.common.datastorage.StorageProperties;
@ -135,10 +138,10 @@ public class RadarDao extends PluginDao {
dataStore.addDataRecord(bdr, sp); dataStore.addDataRecord(bdr, sp);
} }
if (radarRec.getSymbologyData() != null) { Map<RadarDataKey, RadarDataPoint> symData = radarRec.getSymbologyData();
if (symData != null && !symData.isEmpty()) {
byte[] data = DynamicSerializationManager.getManager( byte[] data = DynamicSerializationManager.getManager(
SerializationType.Thrift).serialize( SerializationType.Thrift).serialize(symData);
radarRec.getSymbologyData());
ByteDataRecord bdr = new ByteDataRecord( ByteDataRecord bdr = new ByteDataRecord(
RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data); RadarStoredData.SYM_DATA_ID, radarRec.getDataURI(), data);
bdr.setCorrelationObject(radarRec); bdr.setCorrelationObject(radarRec);
@ -156,10 +159,11 @@ public class RadarDao extends PluginDao {
dataStore.addDataRecord(bdr, sp); dataStore.addDataRecord(bdr, sp);
} }
if (radarRec.getMapProductVals() != null) { Map<MapValues, Map<String, Map<MapValues, String>>> mapProdVals = radarRec
.getMapProductVals();
if (mapProdVals != null && !mapProdVals.isEmpty()) {
byte[] data = DynamicSerializationManager.getManager( byte[] data = DynamicSerializationManager.getManager(
SerializationType.Thrift).serialize( SerializationType.Thrift).serialize(mapProdVals);
radarRec.getMapProductVals());
ByteDataRecord bdr = new ByteDataRecord( ByteDataRecord bdr = new ByteDataRecord(
RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(), RadarStoredData.PRODUCT_VALS_ID, radarRec.getDataURI(),
data); data);
@ -186,19 +190,21 @@ public class RadarDao extends PluginDao {
dataStore.addDataRecord(rec, sp); dataStore.addDataRecord(rec, sp);
} }
if (radarRec.getMapRecordVals() != null) { Map<MapValues, Map<MapValues, String>> mapRecVals = radarRec
.getMapRecordVals();
if (mapRecVals != null && !mapRecVals.isEmpty()) {
byte[] data = DynamicSerializationManager.getManager( byte[] data = DynamicSerializationManager.getManager(
SerializationType.Thrift).serialize( SerializationType.Thrift).serialize(mapRecVals);
radarRec.getMapRecordVals());
ByteDataRecord bdr = new ByteDataRecord( ByteDataRecord bdr = new ByteDataRecord(
RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data); RadarStoredData.RECORD_VALS_ID, radarRec.getDataURI(), data);
bdr.setCorrelationObject(radarRec); bdr.setCorrelationObject(radarRec);
dataStore.addDataRecord(bdr, sp); dataStore.addDataRecord(bdr, sp);
} }
if (radarRec.getStormIDs() != null) { Map<String, RadarDataKey> stormIds = radarRec.getStormIDs();
if (stormIds != null && !stormIds.isEmpty()) {
byte[] data = DynamicSerializationManager.getManager( byte[] data = DynamicSerializationManager.getManager(
SerializationType.Thrift).serialize(radarRec.getStormIDs()); SerializationType.Thrift).serialize(stormIds);
ByteDataRecord bdr = new ByteDataRecord( ByteDataRecord bdr = new ByteDataRecord(
RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data); RadarStoredData.STORM_IDS_ID, radarRec.getDataURI(), data);
bdr.setCorrelationObject(radarRec); bdr.setCorrelationObject(radarRec);

69
edexOsgi/com.raytheon.uf.common.dataplugin.radar/src/com/raytheon/uf/common/dataplugin/radar/util/RadarRecordUtil.java
View file

@ -56,8 +56,9 @@ import com.vividsolutions.jts.geom.Coordinate;
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Aug 11, 2010 mnash Initial creation * Aug 11, 2010 mnash Initial creation
* Dec 28, 2011 11705 gzhang Fix SCAN missing Rows error * Dec 28, 2011 11705 gzhang Fix SCAN missing Rows error
* Mar 18, 2013 1804 bsteffen Remove AlphanumericValues from radar * Mar 19, 2013 1804 bsteffen Reduce useless data stored in radar hdf5
* HDF5. * Mar 19, 2013 1804 bsteffen Remove empty data structures from radar
* hdf5.
* *
* </pre> * </pre>
* *
@ -86,9 +87,8 @@ public class RadarRecordUtil {
.getTheText())) { .getTheText())) {
Map<GraphicBlockValues, String> map = new HashMap<GraphicBlockValues, String>(); Map<GraphicBlockValues, String> map = new HashMap<GraphicBlockValues, String>();
Matcher m = RadarConstants.graphic_block_pattern Matcher m = RadarConstants.graphic_block_pattern
.matcher( .matcher(getNormalizedGBText(((TextSymbolPacket) packets[j])
getNormalizedGBText( ((TextSymbolPacket) packets[j]).getTheText() ) .getTheText()));
);
if (m.find()) { if (m.find()) {
String storm_id = m.group(1).trim(); String storm_id = m.group(1).trim();
map.put(GraphicBlockValues.AZIMUTH, m map.put(GraphicBlockValues.AZIMUTH, m
@ -149,8 +149,7 @@ public class RadarRecordUtil {
String property) { String property) {
String rval = ""; String rval = "";
for (RadarDataKey curLatLon : record.getSymbologyData() for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) {
.keySet()) {
RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon); RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon);
for (Integer type : currPoint.getDisplayGenericPointData().keySet()) { for (Integer type : currPoint.getDisplayGenericPointData().keySet()) {
@ -170,8 +169,10 @@ public class RadarRecordUtil {
/** /**
* Get the GenericDataComponent. * Get the GenericDataComponent.
* *
* @param record The RadarRecord * @param record
* @param featureId The featureId * The RadarRecord
* @param featureId
* The featureId
* *
* @return The GenericDataComponent, or null if no matches * @return The GenericDataComponent, or null if no matches
*/ */
@ -195,11 +196,13 @@ public class RadarRecordUtil {
public static List<String> getDMDFeatureIDs(RadarRecord record) { public static List<String> getDMDFeatureIDs(RadarRecord record) {
List<String> rval = new ArrayList<String>(); List<String> rval = new ArrayList<String>();
SymbologyBlock sb = record.getSymbologyBlock();
for (Layer layer : record.getSymbologyBlock().getLayers()) { if (sb != null) {
for (Layer layer : sb.getLayers()) {
for (SymbologyPacket packet : layer.getPackets()) for (SymbologyPacket packet : layer.getPackets())
rval.addAll(((DMDPacket) packet).getFeatureIDs()); rval.addAll(((DMDPacket) packet).getFeatureIDs());
} }
}
return rval; return rval;
} }
@ -207,8 +210,7 @@ public class RadarRecordUtil {
String featureId) { String featureId) {
Coordinate rval = null; Coordinate rval = null;
AreaComponent currFeature = null; AreaComponent currFeature = null;
for (RadarDataKey curLatLon : record.getSymbologyData() for (RadarDataKey curLatLon : record.getSymbologyData().keySet()) {
.keySet()) {
RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon); RadarDataPoint currPoint = record.getSymbologyData().get(curLatLon);
for (Integer type : currPoint.getDisplayGenericPointData().keySet()) { for (Integer type : currPoint.getDisplayGenericPointData().keySet()) {
@ -419,16 +421,23 @@ public class RadarRecordUtil {
return record.srmSourceName != null; return record.srmSourceName != null;
} }
private static final DHRValues[] ADAP32_VALUES = { private static final DHRValues[] ADAP32_VALUES = { DHRValues.BEAMWIDTH,
DHRValues.BEAMWIDTH, DHRValues.BLOCKAGETHRESHOLD, DHRValues.CLUTTERTHRESHOLD, DHRValues.WEIGHTTHRESHOLD, DHRValues.BLOCKAGETHRESHOLD, DHRValues.CLUTTERTHRESHOLD,
DHRValues.FULLHYBRIDSCANTHRESH, DHRValues.LOWREFLTHRESHOLD, DHRValues.RAINDETREFLTHRESHOLD, DHRValues.RAINDETAREATHRESHOLD, DHRValues.WEIGHTTHRESHOLD, DHRValues.FULLHYBRIDSCANTHRESH,
DHRValues.RAINDETTIMETHRESHOLD, DHRValues.ZRMULTCOEFF, DHRValues.ZRPOWERCOEFF, DHRValues.MINREFLTORATE, DHRValues.LOWREFLTHRESHOLD, DHRValues.RAINDETREFLTHRESHOLD,
DHRValues.MAXREFLTORATE, DHRValues.NUMEXCLZONE, DHRValues.RANGECUTOFF, DHRValues.RANGEEFFCOEFF1, DHRValues.RAINDETAREATHRESHOLD, DHRValues.RAINDETTIMETHRESHOLD,
DHRValues.RANGEEFFCOEFF2, DHRValues.RANGEEFFCOEFF3, DHRValues.MINPRECIPRATEINCL, DHRValues.MAXPRECIPRATEALLOW, DHRValues.ZRMULTCOEFF, DHRValues.ZRPOWERCOEFF,
DHRValues.THRESHELAPSEDTIME, DHRValues.MAXTIMEFORINTERP, DHRValues.MINTIMEHOURLYPERIOD, DHRValues.THRESHOLDHROUTLIER, DHRValues.MINREFLTORATE, DHRValues.MAXREFLTORATE,
DHRValues.ENDTIMEGAGEACCUM, DHRValues.MAXPERIODACCUMVAL, DHRValues.MAXHOURLYACCUMVAL, DHRValues.TIMEBIASEST, DHRValues.NUMEXCLZONE, DHRValues.RANGECUTOFF,
DHRValues.THRESHNOGAGERADAR, DHRValues.RESETBIASVALUE, DHRValues.LONGESTALLOWLAG, DHRValues.BIASAPPLIEDFLAG DHRValues.RANGEEFFCOEFF1, DHRValues.RANGEEFFCOEFF2,
}; DHRValues.RANGEEFFCOEFF3, DHRValues.MINPRECIPRATEINCL,
DHRValues.MAXPRECIPRATEALLOW, DHRValues.THRESHELAPSEDTIME,
DHRValues.MAXTIMEFORINTERP, DHRValues.MINTIMEHOURLYPERIOD,
DHRValues.THRESHOLDHROUTLIER, DHRValues.ENDTIMEGAGEACCUM,
DHRValues.MAXPERIODACCUMVAL, DHRValues.MAXHOURLYACCUMVAL,
DHRValues.TIMEBIASEST, DHRValues.THRESHNOGAGERADAR,
DHRValues.RESETBIASVALUE, DHRValues.LONGESTALLOWLAG,
DHRValues.BIASAPPLIEDFLAG };
public static Map<DHRValues, Double> getDHRValues(RadarRecord record) { public static Map<DHRValues, Double> getDHRValues(RadarRecord record) {
Map<DHRValues, Double> map = new HashMap<DHRValues, Double>(); Map<DHRValues, Double> map = new HashMap<DHRValues, Double>();
@ -477,9 +486,11 @@ public class RadarRecordUtil {
while (vi < nv) { while (vi < nv) {
s = v[vi++]; s = v[vi++];
if (s.equals("SUPL(15)")) { if (s.equals("SUPL(15)")) {
/* // average scan date/time are never used... /*
map.put(DHRValues.AVGSCANDATE, parseDHRValue(text, vi + 0)); * // average scan date/time are never used...
map.put(DHRValues.AVGSCANTIME, parseDHRValue(text, vi + 1)); * map.put(DHRValues.AVGSCANDATE, parseDHRValue(text, vi
* + 0)); map.put(DHRValues.AVGSCANTIME,
* parseDHRValue(text, vi + 1));
*/ */
flagZeroHybrid = (int) parseDHRValue(v[vi + 2]); flagZeroHybrid = (int) parseDHRValue(v[vi + 2]);
if (flagZeroHybrid != 0 && flagZeroHybrid != 1) if (flagZeroHybrid != 0 && flagZeroHybrid != 1)
@ -491,7 +502,8 @@ public class RadarRecordUtil {
} }
} }
} else if (s.equals("ADAP(38)")) { } else if (s.equals("ADAP(38)")) {
// Don't have documentation for older formats, so copying logic from A1 decodeDHR.C. // Don't have documentation for older formats, so copying logic
// from A1 decodeDHR.C.
map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9])); map.put(DHRValues.ZRMULTCOEFF, parseDHRValue(v[vi + 9]));
map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10])); map.put(DHRValues.ZRPOWERCOEFF, parseDHRValue(v[vi + 10]));
map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25])); map.put(DHRValues.MAXPRECIPRATEALLOW, parseDHRValue(v[vi + 25]));
@ -566,7 +578,8 @@ public class RadarRecordUtil {
*/ */
private static String getNormalizedGBText(String text) { private static String getNormalizedGBText(String text) {
if(text == null || text.isEmpty() || ( (! text.contains(">")) && (! text.contains("<")) ) ) if (text == null || text.isEmpty()
|| ((!text.contains(">")) && (!text.contains("<"))))
return text; return text;
/* /*