From ff687e572ba257e127093fa2e8a9825c333aafe6 Mon Sep 17 00:00:00 2001 From: Steve Harris Date: Mon, 18 Mar 2013 12:53:04 -0400 Subject: [PATCH] 13.2.1-15 baseline Former-commit-id: 7617a2437f4a51b739a817aa8ee1fef3c954e041 [formerly a6d7a7ca85cc185d9cd3394a6e6e82f72b8f70fc] Former-commit-id: 7b52d3eb6e6a497af450d4e1bb668bc21ecf7a18 --- .../grid/staticdata/StaticDataGenerator.java | 6 +- .../res/spring/ffmp-ingest.xml | 4 +- .../uf/edex/plugin/ffmp/FFMPGenerator.java | 75 ++++++++++++++++--- .../uf/edex/plugin/grid/dao/GridDao.java | 6 +- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java index 34a84c3e95..6a683aeedc 100644 --- a/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java @@ -70,6 +70,7 @@ import com.raytheon.uf.edex.plugin.grid.dao.GridDao; * ------------ ---------- ----------- -------------------------- * Dec 3, 2010 rjpeter Initial creation * Mar 07, 2013 1587 bsteffen rewrite static data generation. + * Mar 14, 2013 1587 bsteffen Fix persisting to datastore. * * * @@ -267,7 +268,7 @@ public class StaticDataGenerator { for (GridRecord staticRecord : datastoreRecords) { populateMessageData(staticRecord); } - dao.persistToHDF5(databaseRecords.toArray(new PluginDataObject[0])); + dao.persistToHDF5(datastoreRecords.toArray(new PluginDataObject[0])); } if (!databaseRecords.isEmpty()) { dao.persistToDatabase(databaseRecords @@ -423,7 +424,8 @@ public class StaticDataGenerator { datasets = Collections.emptyList(); } } - if (datasets.contains(missing)) { + if (!datasets.contains(staticRecord.getParameter() + .getAbbreviation())) { missing.add(staticRecord); } } diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml index e768457d1a..8092b1e020 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml @@ -8,8 +8,8 @@ - - + + diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java index 1f7c5eafae..e94370e464 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java @@ -118,6 +118,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff; * 07/31/2011 578 dhladky FFTI modifications * 01/27/13 1478 D. Hladky Added creation of full cache records to help read write stress on NAS * 02/25/13 1660 D. Hladky Redesigned data flow for FFTI in order to have only one mosaic piece in memory at a time. + * 03/13/13 1478 D. Hladky non-FFTI mosaic containers weren't getting ejected. Made it so that they are ejected after processing as well. * * * @author dhladky @@ -701,18 +702,47 @@ public class FFMPGenerator extends CompositeProductGenerator implements fftiSources.add(ffmp.getFFTISource()); ffti.processFFTI(); } - - // Do the accumulation now, more memory efficient. + + // Do the accumulation now, more memory efficient. // Only one piece in memory at a time - for (String attribute: ffmp.getAttributes()) { - if (attribute.equals(ATTRIBUTE.ACCUM.getAttribute())) { - FFTIAccum accum = getAccumulationForSite(ffmpProduct.getDisplayName(), siteKey, dataKey, fftiSource.getDurationHour(), ffmpProduct.getUnit(siteKey)); - if (statusHandler.isPriorityEnabled(Priority.DEBUG)) { - statusHandler.debug("Accumulating FFTI for source: "+ffmpProduct.getDisplayName()+" site: "+siteKey+" data: "+dataKey+" duration: "+fftiSource.getDurationHour()+ " accumulation: "+accum.getAccumulation()); + for (String attribute : ffmp.getAttributes()) { + if (attribute.equals(ATTRIBUTE.ACCUM + .getAttribute())) { + FFTIAccum accum = getAccumulationForSite( + ffmpProduct.getDisplayName(), + siteKey, dataKey, + fftiSource.getDurationHour(), + ffmpProduct.getUnit(siteKey)); + if (statusHandler + .isPriorityEnabled(Priority.DEBUG)) { + statusHandler + .debug("Accumulating FFTI for source: " + + ffmpProduct + .getDisplayName() + + " site: " + + siteKey + + " data: " + + dataKey + + " duration: " + + fftiSource + .getDurationHour() + + " accumulation: " + + accum.getAccumulation()); } - } + } } } + + SourceXML source = getSourceConfig().getSource( + ffmpRec.getSourceName()); + + if (!source.getSourceType().equals( + SOURCE_TYPE.GUIDANCE.getSourceType())) { + String sourceSiteDataKey = getSourceSiteDataKey(source, + dataKey, ffmpRec); + ffmpData.remove(sourceSiteDataKey); + statusHandler.info("Removing from memory: "+sourceSiteDataKey); + } } } } @@ -1918,7 +1948,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } - ffmpData.remove(siteDataKey); accumulator.setReset(false); writeFFTIData(siteDataKey, accumulator); } @@ -2102,5 +2131,33 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } + + + /** + * Find siteSourceDataKey + * + * @param source + * @param dataKey + * @param ffmpRec + * @return + */ + private String getSourceSiteDataKey(SourceXML source, String dataKey, FFMPRecord ffmpRec) { + + String sourceName = source.getSourceName(); + String sourceSiteDataKey = null; + + if (source.getSourceType().equals( + SOURCE_TYPE.GUIDANCE.getSourceType())) { + sourceName = source.getDisplayName(); + sourceSiteDataKey = sourceName; + + } else { + sourceName = ffmpRec.getSourceName(); + sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() + + "-" + dataKey; + } + + return sourceSiteDataKey; + } } \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.grid/src/com/raytheon/uf/edex/plugin/grid/dao/GridDao.java b/edexOsgi/com.raytheon.uf.edex.plugin.grid/src/com/raytheon/uf/edex/plugin/grid/dao/GridDao.java index 3beb5c4ad9..2b58fb81de 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.grid/src/com/raytheon/uf/edex/plugin/grid/dao/GridDao.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.grid/src/com/raytheon/uf/edex/plugin/grid/dao/GridDao.java @@ -66,6 +66,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * 4/7/09 1994 bphillip Initial Creation + * Mar 14, 2013 1587 bsteffen Fix static data persisting to datastore. * * * @@ -99,10 +100,13 @@ public class GridDao extends PluginDao { long[] sizes = new long[] { location.getNx(), location.getNy() }; String abbrev = gridRec.getParameter().getAbbreviation(); String group = gridRec.getDataURI(); + String datasetName = "Data"; if (GridPathProvider.STATIC_PARAMETERS.contains(abbrev)) { group = "/" + location.getId(); + datasetName = abbrev; } - AbstractStorageRecord storageRecord = new FloatDataRecord("Data", + AbstractStorageRecord storageRecord = new FloatDataRecord( + datasetName, group, (float[]) messageData, 2, sizes); storageRecord.setCorrelationObject(gridRec);