From 84dae0a6b8d74fa86082505bfe7b73c12b677b1a Mon Sep 17 00:00:00 2001 From: Dave Hladky Date: Wed, 17 Jul 2013 13:27:06 -0500 Subject: [PATCH] Issue #2184 Removed all other HUCs other than ALL from storage Change-Id: I04885f5162cbe25b4345fe797da5a6f7ac246cc5 Former-commit-id: 251cf3cca7c6653300f115083aa860444fee0ea9 --- .../uf/viz/monitor/ffmp/FFMPMonitor.java | 154 +++--- .../uf/viz/monitor/ffmp/FFMPProcessUris.java | 12 +- .../uf/viz/monitor/ffmp/FFMPSourceData.java | 67 +-- .../ffmp/ui/rsc/FFMPDataGenerator.java | 245 ++++---- .../viz/monitor/ffmp/ui/rsc/FFMPResource.java | 187 ++++--- .../monitor/ffmp/ui/rsc/FFMPResourceData.java | 69 +-- .../monitor/ffmp/ui/rsc/FFMPRowGenerator.java | 188 +++++-- .../ffmp/ui/thread/AbstractLoadJob.java | 51 +- .../dataplugin/ffmp/FFMPAggregateRecord.java | 42 +- .../uf/common/dataplugin/ffmp/FFMPBasin.java | 2 +- .../common/dataplugin/ffmp/FFMPBasinData.java | 94 +++- .../dataplugin/ffmp/FFMPDataContainer.java | 162 +++--- .../uf/common/dataplugin/ffmp/FFMPRecord.java | 115 ++-- .../common/dataplugin/ffmp/FFMPTemplates.java | 48 +- .../common/dataplugin/ffmp/dao/FFMPDao.java | 144 ++--- .../ffmp/dataaccess/FFMPGeometryFactory.java | 6 +- .../uf/edex/plugin/ffmp/FFMPGenerator.java | 521 +++++++++--------- .../common/FFMPInterpolatedGuidanceDelay.java | 21 +- .../plugin/ffmp/common/FFMPProcessor.java | 107 +--- .../plugin/ffmp/common/FFTIProcessor.java | 30 +- 20 files changed, 1087 insertions(+), 1178 deletions(-) diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java index bae7775927..2cf1f08dd2 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java @@ -97,6 +97,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener; * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. * Jun 06, 2013 2075 njensen No longer starts loading threads, resourceData does that * Jun 07, 2013 2075 njensen Extracted FFMPProcessUris to separate class + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -291,11 +292,10 @@ public class FFMPMonitor extends ResourceMonitor { * @param phuc * @return */ - protected List getLoadedUris(String siteKey, String source, - String phuc) { + protected List getLoadedUris(String siteKey, String source) { FFMPSiteData siteData = siteDataMap.get(siteKey); FFMPSourceData sourceData = siteData.getSourceData(source); - return sourceData.getLoadedUris(phuc); + return sourceData.getLoadedUris(); } /** @@ -308,7 +308,7 @@ public class FFMPMonitor extends ResourceMonitor { * @throws VizException */ public void populateFFMPRecord(String siteKey, String dataKey, - String source, Date ptime, String phuc, boolean retrieveNew) { + String source, Date ptime, boolean retrieveNew) { if (source != null) { boolean dupOverride = false; @@ -325,11 +325,11 @@ public class FFMPMonitor extends ResourceMonitor { for (String uri : uris) { if (uri != null) { if (dupOverride - || !getLoadedUris(siteKey, source, phuc) + || !getLoadedUris(siteKey, source) .contains(uri)) { try { populateFFMPRecord(siteKey, new FFMPRecord( - uri), source, phuc); + uri), source); } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, "FFMP Can't retrieve FFMP URI, " @@ -353,10 +353,10 @@ public class FFMPMonitor extends ResourceMonitor { * @throws VizException */ public FFMPRecord populateFFMPRecord(String uri, String siteKey, - String source, String phuc) throws Exception { + String source) throws Exception { try { - populateFFMPRecord(siteKey, new FFMPRecord(uri), source, phuc); + populateFFMPRecord(siteKey, new FFMPRecord(uri), source); } catch (VizException e) { statusHandler.handle(Priority.INFO, "FFMP Can't retrieve FFMP URI, " + uri, e); @@ -387,31 +387,26 @@ public class FFMPMonitor extends ResourceMonitor { FFMPRecord curRecord = sourceData.getRecord(); if (curRecord == null) { - // add each huc requested - for (String huc : data.getBasinsMap().keySet()) { - // add all of the uris - for (Entry> duris : uris.entrySet()) { - if (data.getTimes().contains(duris.getKey().getTime())) { - for (String uri : duris.getValue()) { - if (curRecord == null) { - curRecord = new FFMPRecord(uri); - sourceData.setRecord(curRecord); - } - sourceData.addLoadedUri(huc, uri); + // add all of the uris + for (Entry> duris : uris.entrySet()) { + if (data.getTimes().contains(duris.getKey().getTime())) { + for (String uri : duris.getValue()) { + if (curRecord == null) { + curRecord = new FFMPRecord(uri); + sourceData.setRecord(curRecord); } + sourceData.addLoadedUri(uri); } } } } if (curRecord != null) { - for (Entry entry : data.getBasinsMap() - .entrySet()) { - FFMPBasinData basinData = entry.getValue(); - basinData.populate(data.getTimes()); - curRecord.populate(basinData, entry.getKey()); - } + FFMPBasinData basinData = data.getBasins(); + basinData.populate(data.getTimes()); + curRecord.populate(basinData); + } } @@ -422,13 +417,13 @@ public class FFMPMonitor extends ResourceMonitor { * @param siteKey * @param ffmpRec * @param source - * @param phuc + * * @throws Exception */ public void populateFFMPRecord(String siteKey, FFMPRecord ffmpRec, - String source, String phuc) throws Exception { + String source) throws Exception { - FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source, phuc); + FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source); flr.run(); } @@ -446,12 +441,12 @@ public class FFMPMonitor extends ResourceMonitor { String source, String phuc, FFMPBasin basin) throws VizException { if (dataUri != null) { - List uris = getLoadedUris(siteKey, source, phuc); + List uris = getLoadedUris(siteKey, source); if (!uris.contains(dataUri)) { try { SourceXML sourceXML = fscm.getSource(source); FFMPRecord ffmpRec = populateFFMPRecord(dataUri, siteKey, - source, phuc); + source); File loc = HDF5Util.findHDF5Location(ffmpRec); IDataStore dataStore = DataStoreFactory.getDataStore(loc); @@ -698,14 +693,12 @@ public class FFMPMonitor extends ResourceMonitor { * */ public FFMPRecord getFFMPData(ProductXML product, String siteKey, - String dataKey, String sourceName, Date ptime, String phuc, - boolean retrieveNew) { + String dataKey, String sourceName, Date ptime, boolean retrieveNew) { FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName) .getRecord(); - if ((record != null) - && (record.getBasinData(phuc).getBasins().size() > 0)) { + if ((record != null) && (record.getBasinData().getBasins().size() > 0)) { SourceXML sourceXML = getSourceConfig().getSource(sourceName); @@ -731,8 +724,7 @@ public class FFMPMonitor extends ResourceMonitor { continue; } else { populateFFMPRecord(siteKey, dataKey, - source.getSourceName(), ptime, phuc, - retrieveNew); + source.getSourceName(), ptime, retrieveNew); } } } else { @@ -740,7 +732,7 @@ public class FFMPMonitor extends ResourceMonitor { if (!siteDataMap.get(siteKey).getSourceData(sourceName) .hasLoadedAnyUris()) { populateFFMPRecord(siteKey, dataKey, sourceName, ptime, - phuc, retrieveNew); + retrieveNew); } } @@ -748,7 +740,7 @@ public class FFMPMonitor extends ResourceMonitor { .getRecord(); } else { - populateFFMPRecord(siteKey, dataKey, sourceName, ptime, phuc, + populateFFMPRecord(siteKey, dataKey, sourceName, ptime, retrieveNew); } @@ -775,17 +767,17 @@ public class FFMPMonitor extends ResourceMonitor { .getGuidanceSources(product, guidSrc)) { populateFFMPRecord(siteKey, dataKey, - ffgSource.getSourceName(), ptime, phuc, + ffgSource.getSourceName(), ptime, retrieveNew); } } else { populateFFMPRecord(siteKey, dataKey, sourceName, - ptime, phuc, retrieveNew); + ptime, retrieveNew); } } } else { populateFFMPRecord(siteKey, dataKey, sourceName, ptime, - phuc, retrieveNew); + retrieveNew); } } else { // special case where FFG is the primary source @@ -799,7 +791,7 @@ public class FFMPMonitor extends ResourceMonitor { sourceName = sourcexml.getDisplayName(); } else { populateFFMPRecord(siteKey, dataKey, sourceName, ptime, - phuc, retrieveNew); + retrieveNew); } } @@ -1054,20 +1046,18 @@ public class FFMPMonitor extends ResourceMonitor { * Get the pertinent QPE source Record. * * @param date - * @param phuc * @param retrieveNew * @return */ public FFMPRecord getQPERecord(ProductXML product, String siteKey, - String dataKey, String sourceName, Date date, String phuc, - boolean retrieveNew) { + String dataKey, String sourceName, Date date, boolean retrieveNew) { // comparisons done with table display if (product != null) { sourceName = product.getQpe(); } - return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc, + return getFFMPData(product, siteKey, dataKey, sourceName, date, retrieveNew); } @@ -1095,12 +1085,11 @@ public class FFMPMonitor extends ResourceMonitor { * Get the rate record. * * @param date - * @param phuc * @param retrieveNew * @return */ public FFMPRecord getRateRecord(ProductXML product, String siteKey, - String dataKey, String sourceName, Date date, String phuc, + String dataKey, String sourceName, Date date, boolean retrieveNew) { // comparisons done with table display @@ -1108,7 +1097,7 @@ public class FFMPMonitor extends ResourceMonitor { sourceName = product.getRate(); } - return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc, + return getFFMPData(product, siteKey, dataKey, sourceName, date, retrieveNew); } @@ -1142,8 +1131,7 @@ public class FFMPMonitor extends ResourceMonitor { * @return */ public FFMPRecord getQPFRecord(ProductXML product, String siteKey, - String dataKey, String sourceName, Date date, String phuc, - boolean retrieveNew) { + String dataKey, String sourceName, Date date, boolean retrieveNew) { FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance() .getTableConfigData(siteKey); @@ -1157,7 +1145,7 @@ public class FFMPMonitor extends ResourceMonitor { .getSourceName(); } - return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc, + return getFFMPData(product, siteKey, dataKey, sourceName, date, retrieveNew); } @@ -1210,8 +1198,7 @@ public class FFMPMonitor extends ResourceMonitor { sourceName = source.getDisplayName(); } - return getFFMPData(product, siteKey, null, sourceName, date, phuc, - false); + return getFFMPData(product, siteKey, null, sourceName, date, false); } /** @@ -1226,7 +1213,7 @@ public class FFMPMonitor extends ResourceMonitor { * @return */ public Map getGuidanceRecords(ProductXML product, - String siteKey, Date date, String phuc, boolean retrieveNew) { + String siteKey, Date date, boolean retrieveNew) { Map guidRecs = new HashMap(); ProductRunXML productRun = FFMPRunConfigurationManager.getInstance() @@ -1236,7 +1223,7 @@ public class FFMPMonitor extends ResourceMonitor { for (String type : guidTypes) { FFMPRecord guidRec = getFFMPData(product, siteKey, null, type, - date, phuc, retrieveNew); + date, retrieveNew); guidRecs.put(type, guidRec); } @@ -1275,7 +1262,7 @@ public class FFMPMonitor extends ResourceMonitor { * @return */ public FFMPRecord getVirtualRecord(ProductXML product, String siteKey, - String dataKey, String sourceName, Date date, String phuc, + String dataKey, String sourceName, Date date, boolean retrieveNew) { // comparisons done with table display // field doesn't matter here @@ -1284,7 +1271,7 @@ public class FFMPMonitor extends ResourceMonitor { sourceName = product.getVirtual(); } - return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc, + return getFFMPData(product, siteKey, dataKey, sourceName, date, retrieveNew); } @@ -1305,7 +1292,7 @@ public class FFMPMonitor extends ResourceMonitor { FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName) .getRecord(); if (record != null) { - FFMPBasinData basinData = record.getBasinData(phuc); + FFMPBasinData basinData = record.getBasinData(); if (basinData != null) { basin = basinData.get(pfaf); } @@ -1350,7 +1337,7 @@ public class FFMPMonitor extends ResourceMonitor { .get(siteKey) .getSourceData( source.getSourceName()) - .getLoadedUris(phuc).contains(uri)) { + .getLoadedUris().contains(uri)) { // populate point only populateFFMPBasin(uri, siteKey, source.getSourceName(), phuc, @@ -1376,7 +1363,7 @@ public class FFMPMonitor extends ResourceMonitor { for (String uri : uris) { if (!siteDataMap.get(siteKey) .getSourceData(sourceName) - .getLoadedUris(phuc).contains(uri)) { + .getLoadedUris().contains(uri)) { // populate point only populateFFMPBasin(uri, siteKey, sourceName, phuc, fgb); @@ -1397,7 +1384,7 @@ public class FFMPMonitor extends ResourceMonitor { POINT_RETRIVAL: for (List uris : availableUris.values()) { for (String uri : uris) { if (!siteDataMap.get(siteKey).getSourceData(sourceName) - .getLoadedUris(phuc).contains(uri)) { + .getLoadedUris().contains(uri)) { // populate point only populateFFMPBasin(uri, siteKey, sourceName, phuc, basin); @@ -1593,21 +1580,15 @@ public class FFMPMonitor extends ResourceMonitor { .hasLoadedAnyUris()) { FFMPSourceData sourceData = siteData .getSourceData(sourceName); - Set hucs = sourceData - .getLoadedHucs(); - for (String huc : hucs) { - sourceData.getLoadedUris(huc).remove( - uri); - } + + sourceData.removeLoadedUri(uri); } } } else { FFMPSourceData sourceData = siteData .getSourceData(fsource); - Set hucs = sourceData.getLoadedHucs(); - for (String huc : hucs) { - sourceData.getLoadedUris(huc).remove(uri); - } + + sourceData.removeLoadedUri(uri); } } } @@ -1625,16 +1606,14 @@ public class FFMPMonitor extends ResourceMonitor { * @param siteKey * @param sourceName * @param barrierTime - * @param phuc */ public void processUri(String uri, String siteKey, String sourceName, - Date barrierTime, String phuc) { + Date barrierTime) { if (uri != null) { try { - FFMPRecord record = populateFFMPRecord(uri, siteKey, - sourceName, phuc); + FFMPRecord record = populateFFMPRecord(uri, siteKey, sourceName); if (record != null) { - record.getBasinData(phuc).loadNow(); + record.getBasinData().loadNow(); SourceXML source = getSourceConfig().getSource(sourceName); if (source != null) { record.setExpiration(source @@ -1656,10 +1635,10 @@ public class FFMPMonitor extends ResourceMonitor { * @param sourceName */ public void processUris(NavigableMap> uriMap, - String siteKey, String sourceName, Date barrierTime, String phuc, + String siteKey, String sourceName, Date barrierTime, SubMonitor smonitor) { FFMPProcessUris processor = new FFMPProcessUris(this, uriMap, siteKey, - sourceName, barrierTime, phuc); + sourceName, barrierTime); processor.run(smonitor); } @@ -2126,16 +2105,13 @@ public class FFMPMonitor extends ResourceMonitor { final String fsource; - final String fhuc; - final String fsiteKey; - public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec, - String source, String huc) throws Exception { + public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec, String source) + throws Exception { this.fffmpRec = ffmpRec; this.fsource = source; this.fsiteKey = siteKey; - this.fhuc = huc; } public void run() { @@ -2143,7 +2119,7 @@ public class FFMPMonitor extends ResourceMonitor { load(); } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, "FFMP load FFMPData, " - + fsource + " " + fhuc, e); + + fsource, e); } } @@ -2151,7 +2127,7 @@ public class FFMPMonitor extends ResourceMonitor { if (fffmpRec != null) { - List uris = getLoadedUris(fsiteKey, fsource, fhuc); + List uris = getLoadedUris(fsiteKey, fsource); String dataUri = fffmpRec.getDataURI(); if (!uris.contains(dataUri)) { Date refTime = fffmpRec.getDataTime().getRefTime(); @@ -2188,20 +2164,20 @@ public class FFMPMonitor extends ResourceMonitor { try { - if (isGageSource && fhuc.equals(FFMPRecord.ALL)) { + if (isGageSource) { curRecord.retrieveVirtualMapFromDataStore(loc, dataUri, getTemplates(fsiteKey), refTime, fffmpRec.getSourceName()); } else { curRecord.retrieveMapFromDataStore(loc, dataUri, - getTemplates(fffmpRec.getSiteKey()), fhuc, + getTemplates(fffmpRec.getSiteKey()), refTime, fffmpRec.getSourceName()); } } catch (Exception e) { statusHandler.handle(Priority.PROBLEM, "FFMP Can't retrieve FFMP URI, " + dataUri, e); } - sourceData.addLoadedUri(fhuc, dataUri); + sourceData.addLoadedUri(dataUri); } } } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java index 5da9a9ea9d..69b063a144 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java @@ -44,6 +44,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 7, 2013 njensen Initial creation + * Jul 15, 2013 2184 dhladky Removed all HUC's but ALL * * * @@ -64,18 +65,15 @@ public class FFMPProcessUris { private final Date fbarrierTime; - private final String fhuc; - private final FFMPMonitor ffmpMonitor; public FFMPProcessUris(FFMPMonitor ffmpMonitor, NavigableMap> uriMap, String siteKey, - String sourceName, Date barrierTime, String phuc) { + String sourceName, Date barrierTime) { this.furiMap = uriMap; this.fsiteKey = siteKey; this.fbarrierTime = barrierTime; this.fsourceName = sourceName; - this.fhuc = phuc; this.ffmpMonitor = ffmpMonitor; } @@ -90,7 +88,7 @@ public class FFMPProcessUris { isGuidance = true; } List loadedUris = ffmpMonitor.getLoadedUris(fsiteKey, - fsourceName, fhuc); + fsourceName); Set populatedRecords = new HashSet(); for (List uris : furiMap.descendingMap().values()) { for (String uri : uris) { @@ -102,7 +100,7 @@ public class FFMPProcessUris { || isGuidance) { try { record = ffmpMonitor.populateFFMPRecord(uri, - fsiteKey, fsourceName, fhuc); + fsiteKey, fsourceName); if (record != null) { populatedRecords.add(record); if (source != null) { @@ -121,7 +119,7 @@ public class FFMPProcessUris { monitor.beginTask(null, populatedRecords.size()); for (FFMPRecord record : populatedRecords) { - record.getBasinData(fhuc).loadNow(); + record.getBasinData().loadNow(); monitor.worked(1); } } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java index 3deb125405..c233effb25 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java @@ -20,11 +20,9 @@ package com.raytheon.uf.viz.monitor.ffmp; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentNavigableMap; import java.util.concurrent.ConcurrentSkipListMap; @@ -42,6 +40,7 @@ import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord; * ------------ ---------- ----------- -------------------------- * Feb 18, 2013 njensen Initial creation * Feb 28, 2013 1729 dhladky Sped up, synch blocks were hanging it. + * Jul 15, 2013 2184 dhladky Removed all HUC's but ALL * * * @@ -59,7 +58,7 @@ public class FFMPSourceData { private ConcurrentNavigableMap> availableUris = new ConcurrentSkipListMap>(); /** map of huc to list of loaded URIs **/ - private ConcurrentMap> loadedUris = new ConcurrentHashMap>(); + private List loadedUris = new ArrayList(); /** * Clears the data @@ -68,7 +67,9 @@ public class FFMPSourceData { ffmpData = null; previousUriQueryDate = null; availableUris.clear(); - loadedUris.clear(); + synchronized (loadedUris) { + loadedUris.clear(); + } } /** @@ -110,45 +111,23 @@ public class FFMPSourceData { } /** - * Gets the URIs associated with a HUC that have been loaded. + * Gets the URIs that have been loaded. * - * @param huc * @return */ - public List getLoadedUris(String huc) { - - List loaded = loadedUris.get(huc); - - if (loaded == null) { - loaded = new ArrayList(); - List previous = loadedUris.putIfAbsent(huc, loaded); - if (previous != null) { - return previous; - } - } - - return loaded; + public List getLoadedUris() { + return Collections.unmodifiableList(loadedUris); } /** - * Tracks a URI associated with a HUC as loaded. + * Tracks a URI as loaded. * - * @param huc * @param uri */ - public void addLoadedUri(String huc, String uri) { - - List uriList = loadedUris.get(huc); - - if (uriList == null) { - uriList = new ArrayList(); - List previous = loadedUris.putIfAbsent(huc, uriList); - if (previous != null) { - uriList = previous; - } + public void addLoadedUri(String uri) { + synchronized (loadedUris) { + loadedUris.add(uri); } - - uriList.add(uri); } /** @@ -160,15 +139,6 @@ public class FFMPSourceData { return !loadedUris.isEmpty(); } - /** - * Gets the set of HUCs that have loaded some URIs. - * - * @return - */ - public Set getLoadedHucs() { - return loadedUris.keySet(); - } - /** * Gets the Available URIs based on time. * @@ -178,4 +148,15 @@ public class FFMPSourceData { return availableUris; } + /** + * Removes a URI + * + * @param uri + */ + public void removeLoadedUri(String uri) { + synchronized (loadedUris) { + loadedUris.remove(uri); + } + } + } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java index 01827ef296..a70a7ad073 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java @@ -77,6 +77,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData; * May 22, 2013 1902 mpduff Code cleanup. * Jun 11, 2013 2085 njensen Extracted row creation to FFMPRowGenerator and * multi-threaded row creation. + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -164,134 +165,139 @@ public class FFMPDataGenerator { public FFMPTableData generateFFMPData() throws Exception { // You should always have at least a QPE data source FFMPTableData tData = null; - // update the FFFGDataManager FFFGDataMgr.getUpdatedInstance(); try { + FIELDS field = getBaseField(); if (field == null || baseRec == null) { return tData; } - FFMPBasinData fbd = null; - if (centeredAggregationKey != null) { - fbd = baseRec.getBasinData(FFMPRecord.ALL); - } else { - fbd = baseRec.getBasinData(huc); - } - - tData = new FFMPTableData(fbd.getBasins().size()); + List domains = resource.getDomains(); - if (!fbd.getBasins().isEmpty()) { - if ((centeredAggregationKey == null) - || huc.equals(FFMPRecord.ALL)) { - // System.out.println(fbd.getBasins().keySet().size() - // + " rows in the table"); + + if ((centeredAggregationKey == null) || huc.equals(FFMPRecord.ALL)) { + // System.out.println(fbd.getBasins().keySet().size() + // + " rows in the table"); + if (huc.equals(FFMPRecord.ALL)) { + + FFMPBasinData fbd = baseRec.getBasinData(); + tData = new FFMPTableData(fbd.getBasins().size()); + for (Long key : fbd.getBasins().keySet()) { - if (huc.equals(FFMPRecord.ALL)) { - FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key); - if (fmdb == null) { - continue; - } - for (DomainXML domain : domains) { - String cwa = domain.getCwa(); + FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key); + if (fmdb == null) { + continue; + } - if ((cwa.equals(fmdb.getCwa())) - || (domain.isPrimary() && fmdb - .isPrimaryCwa())) { - try { - setFFMPRow(fbd.get(key), tData, false, - cwa); - } catch (Exception e) { - statusHandler.handle(Priority.PROBLEM, - "Couldn't create table row", e); - } - if (virtualBasin != null) { - for (Long id : ft - .getVirtualGageBasinLookupIds( - siteKey, key, huc, - resource.basinTableDlg - .getRowName())) { - try { - setFFMPRow( - virtualBasin.get(id), - tData, true, cwa); - } catch (Exception e) { - statusHandler.handle( - Priority.PROBLEM, - "Couldn't create table row" - + e); - } - } - } + for (DomainXML domain : domains) { + String cwa = domain.getCwa(); + + if ((cwa.equals(fmdb.getCwa())) + || (domain.isPrimary() && fmdb + .isPrimaryCwa())) { + try { + setFFMPRow(fbd.get(key), tData, false, cwa); + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + "Couldn't create table row", e); } - } - - } else { - /* - * make sure at least one basin in the agg is in the - * CWA - */ - - List pfafs = ft.getAggregatePfafs(key, - siteKey, huc); - - boolean isVGB = false; - if (ft.checkVGBsInAggregate(key, siteKey, huc)) { - isVGB = true; - } - - if (!pfafs.isEmpty()) { - - FFMPBasinMetaData fmdb = ft.getBasinInDomains( - siteKey, domains, pfafs); - - if (fmdb != null) { - try { - setFFMPRow(fbd.get(key), tData, isVGB, - null); - } catch (Exception e) { - statusHandler.handle(Priority.PROBLEM, - "Couldn't create table row", e); + if (virtualBasin != null) { + for (Long id : ft + .getVirtualGageBasinLookupIds( + siteKey, key, huc, + resource.basinTableDlg + .getRowName())) { + try { + setFFMPRow(virtualBasin.get(id), + tData, true, cwa); + } catch (Exception e) { + statusHandler.handle( + Priority.PROBLEM, + "Couldn't create table row" + + e); + } } } } } } } - // show pfafs in aggregation + else { - for (Long key : resource.getCenteredAggregatePfafs()) { + + // Find all of the basins for this HUC level + List keyList = ft + .getHucKeyList(siteKey, huc, domains); + tData = new FFMPTableData(keyList.size()); - FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key); + for (Long key : keyList) { - if (fmdb != null) { - for (DomainXML domain : domains) { + List pfafs = ft.getAggregatePfafs(key, siteKey, + huc); - if ((domain.getCwa().equals(fmdb.getCwa())) - || (domain.isPrimary() && fmdb - .isPrimaryCwa())) { + boolean isVGB = false; + if (ft.checkVGBsInAggregate(key, siteKey, huc)) { + isVGB = true; + } - setFFMPRow(fbd.get(key), tData, false, null); + if (!pfafs.isEmpty()) { - if (virtualBasin != null) { - for (Long id : ft - .getVirtualGageBasinLookupIds( - siteKey, key, huc, - resource.basinTableDlg - .getRowName())) { - try { - setFFMPRow( - virtualBasin.get(id), - tData, true, null); - } catch (Exception e) { - statusHandler - .handle(Priority.PROBLEM, - "Couldn't create table row", - e); - } + FFMPBasinMetaData fmdb = ft.getBasinInDomains( + siteKey, domains, pfafs); + + if (fmdb != null) { + try { + FFMPBasin basin = new FFMPBasin(key, true); + setFFMPRow(basin, tData, isVGB, null); + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + "Couldn't create table row", e); + } + } + } + + } + } + + } + + // show pfafs in aggregation + else { + + FFMPBasinData fbd = baseRec.getBasinData(); + tData = new FFMPTableData(resource.getCenteredAggregatePfafs().size()); + + for (Long key : resource.getCenteredAggregatePfafs()) { + + FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key); + + if (fmdb != null) { + for (DomainXML domain : domains) { + + if ((domain.getCwa().equals(fmdb.getCwa())) + || (domain.isPrimary() && fmdb + .isPrimaryCwa())) { + + setFFMPRow(fbd.get(key), tData, false, null); + + if (virtualBasin != null) { + for (Long id : ft + .getVirtualGageBasinLookupIds( + siteKey, key, FFMPRecord.ALL, + resource.basinTableDlg + .getRowName())) { + try { + setFFMPRow(virtualBasin.get(id), + tData, true, null); + } catch (Exception e) { + statusHandler + .handle(Priority.PROBLEM, + "Couldn't create table row", + e); } } } @@ -333,7 +339,6 @@ public class FFMPDataGenerator { Date tableTime = resource.getTableTime(); FIELDS field = null; - String localHuc = null; FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance() .getTableConfigData(siteKey); @@ -358,37 +363,27 @@ public class FFMPDataGenerator { monitor.setQpeWindow(new FFMPTimeWindow(tableTime, qpeTime)); - if (isWorstCase || (centeredAggregationKey != null)) { - // make sure that "ALL" is loaded - localHuc = FFMPRecord.ALL; - } else { - localHuc = huc; - } - FFMPRecord rateRecord = monitor.getRateRecord(product, siteKey, - dataKey, product.getRate(), paintRefTime, localHuc, true); + dataKey, product.getRate(), paintRefTime, true); FFMPRecord qpeRecord = monitor.getQPERecord(product, siteKey, dataKey, - product.getQpe(), tableTime, localHuc, true); + product.getQpe(), tableTime, true); FFMPRecord qpfRecord = monitor.getQPFRecord(product, siteKey, dataKey, - null, paintRefTime, localHuc, true); + null, paintRefTime, true); guidRecords = monitor.getGuidanceRecords(product, siteKey, tableTime, - localHuc, true); - FFMPRecord virtualRecord = null; - if (localHuc.equals(FFMPRecord.ALL)) { - virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey, - product.getVirtual(), tableTime, localHuc, true); - } - + true); + FFMPRecord virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey, + product.getVirtual(), tableTime, true); + try { if (rateRecord != null) { - rateBasin = rateRecord.getBasinData(localHuc); + rateBasin = rateRecord.getBasinData(); if (!rateBasin.getBasins().isEmpty()) { field = FIELDS.RATE; baseRec = rateRecord; } } if (qpeRecord != null) { - qpeBasin = qpeRecord.getBasinData(localHuc); + qpeBasin = qpeRecord.getBasinData(); if (!qpeBasin.getBasins().isEmpty()) { field = FIELDS.QPE; if (baseRec == null) { @@ -397,21 +392,21 @@ public class FFMPDataGenerator { } } if (qpfRecord != null) { - qpfBasin = qpfRecord.getBasinData(localHuc); + qpfBasin = qpfRecord.getBasinData(); } if (guidRecords != null) { guidBasins = new HashMap(); for (String type : guidRecords.keySet()) { if (guidRecords.get(type) != null) { guidBasins.put(type, guidRecords.get(type) - .getBasinData(localHuc)); + .getBasinData()); } else { guidBasins.put(type, null); } } } if (virtualRecord != null) { - virtualBasin = virtualRecord.getBasinData(localHuc); + virtualBasin = virtualRecord.getBasinData(); } // Get interpolators diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java index 666aff8afb..dcc586607f 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java @@ -169,6 +169,8 @@ import com.vividsolutions.jts.geom.Point; * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. * Jun 06, 2013 2075 njensen No longer schedules load threads, * refactored updates + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL + * * * * @@ -598,15 +600,12 @@ public class FFMPResource extends private FFMPBasin getBasin(Long key, FFMPRecord.FIELDS bfield, Date recentTime, boolean aggregate) throws VizException { FFMPBasin basin = null; - - String huc = null; if (aggregate) { - huc = getHuc(); + basin = new FFMPBasin(key, aggregate); } else { - huc = FFMPRecord.ALL; + basin = getRecord(bfield, recentTime).getBasinData().getBasins() + .get(key); } - basin = getRecord(bfield, recentTime).getBasinData(huc).getBasins() - .get(key); return basin; } @@ -680,14 +679,14 @@ public class FFMPResource extends break; } case RATE: { - value = getRateRecord(recentTime).getBasinData( - FFMPRecord.ALL).getMaxValue(pfafs, recentTime); + value = getRateRecord(recentTime).getBasinData() + .getMaxValue(pfafs, recentTime); break; } case QPF: { - value = getQpfRecord(recentTime).getBasinData( - FFMPRecord.ALL).getAverageMaxValue(pfafs, - recentTime, getQpfSourceExpiration()); + value = getQpfRecord(recentTime).getBasinData() + .getAverageMaxValue(pfafs, recentTime, + getQpfSourceExpiration()); break; } case GUIDANCE: { @@ -695,21 +694,17 @@ public class FFMPResource extends .getCountyFipsByPfaf(pfafs.get(0)); value = getGuidanceRecord() - .getBasinData(FFMPRecord.ALL) + .getBasinData() .getMaxGuidanceValue(pfafs, getGuidanceInterpolation(getFFGName()), getGuidSourceExpiration(getFFGName()), fips); break; } case QPE: { - value = getQpeRecord().getBasinData(FFMPRecord.ALL) - .getAccumMaxValue( - pfafs, - recentTime, - getTableTime(), - getQpeSourceExpiration(), - getResourceData().getPrimarySourceXML() - .isRate()); + value = getQpeRecord().getBasinData().getAccumMaxValue( + pfafs, recentTime, getTableTime(), + getQpeSourceExpiration(), + getResourceData().getPrimarySourceXML().isRate()); break; } } @@ -737,32 +732,64 @@ public class FFMPResource extends break; } case RATE: - value = getBasin(key, field, recentTime, aggregate) - .getValue(recentTime); + if (aggregate) { + value = getRateRecord(recentTime).getBasinData() + .getAverageValue(pfafs, recentTime); + } else { + value = getBasin(key, field, recentTime, aggregate) + .getValue(recentTime); + } + break; case QPF: { - value = getBasin(key, field, recentTime, aggregate) - .getAverageValue(recentTime, - getQpfSourceExpiration()); + if (aggregate) { + value = getQpfRecord(recentTime).getBasinData() + .getAverageValue(pfafs, recentTime, + getQpfSourceExpiration()); + } else { + value = getBasin(key, field, recentTime, aggregate) + .getAverageValue(recentTime, + getQpfSourceExpiration()); + } break; } case GUIDANCE: { - - value = getGuidanceValue( - (FFMPGuidanceBasin) getBasin(key, field, - recentTime, aggregate), recentTime, - getFFGName()); - + if (aggregate) { + getGuidanceRecord() + .getBasinData() + .getAverageGuidanceValue( + pfafs, + getGuidanceInterpolation(getFFGName()), + getGuidSourceExpiration(getFFGName())); + } else { + value = getGuidanceValue( + (FFMPGuidanceBasin) getBasin(key, field, + recentTime, aggregate), recentTime, + getFFGName()); + } break; } case QPE: { - value = getBasin(key, field, recentTime, aggregate) - .getAccumValue( - getTableTime(), - recentTime, - getQpeSourceExpiration(), - getResourceData().getPrimarySourceXML() - .isRate()); + if (aggregate) { + value = getQpeRecord().getBasinData() + .getAccumAverageValue( + pfafs, + getTableTime(), + recentTime, + getQpeSourceExpiration(), + getResourceData() + .getPrimarySourceXML() + .isRate()); + } else { + value = getBasin(key, field, recentTime, aggregate) + .getAccumValue( + getTableTime(), + recentTime, + getQpeSourceExpiration(), + getResourceData() + .getPrimarySourceXML() + .isRate()); + } break; } } @@ -810,17 +837,15 @@ public class FFMPResource extends boolean forced = forceResult.isForced(); if ((forcedPfafs.size() > 0) && forced) { // Recalculate the guidance using the forced value(s) - value = guidRecord.getBasinData(FFMPRecord.ALL) - .getAverageGuidanceValue(pfafList, - this.getGuidanceInterpolation(ffgType), - new Float(value), forcedPfafs, - getGuidSourceExpiration(ffgType)); + value = guidRecord.getBasinData().getAverageGuidanceValue( + pfafList, this.getGuidanceInterpolation(ffgType), + new Float(value), forcedPfafs, + getGuidSourceExpiration(ffgType)); } else if (forcedPfafs.size() > 0) { - value = guidRecord.getBasinData(FFMPRecord.ALL) - .getAverageGuidanceValue(pfafList, - this.getGuidanceInterpolation(ffgType), - Float.NaN, forcedPfafs, - getGuidSourceExpiration(ffgType)); + value = guidRecord.getBasinData().getAverageGuidanceValue( + pfafList, this.getGuidanceInterpolation(ffgType), + Float.NaN, forcedPfafs, + getGuidSourceExpiration(ffgType)); } } @@ -906,10 +931,8 @@ public class FFMPResource extends if ((rateRecord == null) && isNewRate) { try { - String huc = getHucIfWorstCase(); rateRecord = monitor.getRateRecord(getProduct(), getSiteKey(), - getDataKey(), getPrimarySource(), recentTime, huc, - false); + getDataKey(), getPrimarySource(), recentTime, false); isNewRate = false; } catch (Exception e) { e.printStackTrace(); @@ -926,10 +949,9 @@ public class FFMPResource extends public FFMPRecord getQpeRecord() { try { if ((qpeRecord == null) && (getTableTime() != null) && isNewQpe) { - String huc = getHucIfWorstCase(); - qpeRecord = monitor.getQPERecord(getProduct(), getSiteKey(), - getDataKey(), getPrimarySource(), getTableTime(), huc, - false); + qpeRecord = monitor + .getQPERecord(getProduct(), getSiteKey(), getDataKey(), + getPrimarySource(), getTableTime(), false); isNewQpe = false; } } catch (Exception e) { @@ -992,9 +1014,8 @@ public class FFMPResource extends } } - String huc = getHucIfWorstCase(); qpfRecord = monitor.getQPFRecord(getProduct(), getSiteKey(), - getDataKey(), getPrimarySource(), date, huc, false); + getDataKey(), getPrimarySource(), date, false); isNewQpf = false; } } catch (Exception e) { @@ -1014,7 +1035,7 @@ public class FFMPResource extends if ((virtualRecord == null) && isNewVirtual) { virtualRecord = monitor.getVirtualRecord(getProduct(), getSiteKey(), getDataKey(), getPrimarySource(), - getTableTime(), FFMPRecord.ALL, false); + getTableTime(), false); isNewVirtual = false; } @@ -1801,13 +1822,13 @@ public class FFMPResource extends List guids = null; if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) { - qpes = getQpeRecord().getBasinData(FFMPRecord.ALL) + qpes = getQpeRecord().getBasinData() .getAccumValues(pfafs, getTableTime(), recentTime, getQpeSourceExpiration(), isRate()); guids = getGuidanceRecord() - .getBasinData(FFMPRecord.ALL) + .getBasinData() .getGuidanceValues(pfafs, getGuidanceInterpolation(ffgType), getGuidSourceExpiration(ffgType)); @@ -1818,20 +1839,19 @@ public class FFMPResource extends } else { if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) { - qpe = getQpeRecord() - .getBasinData(getHuc()) - .get(key) - .getAccumValue( + qpe = getQpeRecord().getBasinData() + .getAccumAverageValue( + pfafs, getTableTime(), recentTime, getQpeSourceExpiration(), getResourceData().getPrimarySourceXML() .isRate()); - guid = getGuidanceValue( - (FFMPGuidanceBasin) getGuidanceRecord() - .getBasinData(getHuc()).get(key), - recentTime, ffgType); + guid = getGuidanceRecord().getBasinData() + .getAverageGuidanceValue(pfafs, + getGuidanceInterpolation(ffgType), + getGuidSourceExpiration(ffgType)); diff = FFMPUtils.getDiffValue(qpe, guid); } @@ -1839,14 +1859,14 @@ public class FFMPResource extends } else { if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) { qpe = getQpeRecord() - .getBasinData(FFMPRecord.ALL) + .getBasinData() .get(key) .getAccumValue(getTableTime(), recentTime, getQpeSourceExpiration(), isRate()); guid = getGuidanceValue( (FFMPGuidanceBasin) getGuidanceRecord() - .getBasinData(FFMPRecord.ALL).get(key), + .getBasinData().get(key), recentTime, ffgType); guid = forceValue(pfafs, getBasin(key, getField(), recentTime, aggregate), @@ -1881,14 +1901,14 @@ public class FFMPResource extends List qpes = null; List guids = null; if (getQpeRecord() != null) { - qpes = getQpeRecord().getBasinData(FFMPRecord.ALL) + qpes = getQpeRecord().getBasinData() .getAccumValues(pfafs, getTableTime(), recentTime, getQpeSourceExpiration(), isRate()); } if (getGuidanceRecord() != null) { guids = getGuidanceRecord() - .getBasinData(FFMPRecord.ALL) + .getBasinData() .getGuidanceValues(pfafs, getGuidanceInterpolation(ffgType), getGuidSourceExpiration(ffgType)); @@ -1899,32 +1919,31 @@ public class FFMPResource extends } else { if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) { - qpe = getQpeRecord() - .getBasinData(getHuc()) - .get(key) - .getAccumValue( + qpe = getQpeRecord().getBasinData() + .getAccumAverageValue( + pfafs, getTableTime(), recentTime, getQpeSourceExpiration(), getResourceData().getPrimarySourceXML() .isRate()); - guid = getGuidanceValue( - (FFMPGuidanceBasin) getGuidanceRecord() - .getBasinData(getHuc()).get(key), - recentTime, ffgType); - ratio = FFMPUtils.getRatioValue(qpe, guid); + + guid = getGuidanceRecord().getBasinData() + .getAverageGuidanceValue(pfafs, + getGuidanceInterpolation(ffgType), + getGuidSourceExpiration(ffgType)); } } } else { if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) { qpe = getQpeRecord() - .getBasinData(FFMPRecord.ALL) + .getBasinData() .get(key) .getAccumValue(getTableTime(), recentTime, getQpeSourceExpiration(), isRate()); guid = getGuidanceValue( (FFMPGuidanceBasin) getGuidanceRecord() - .getBasinData(FFMPRecord.ALL).get(key), + .getBasinData().get(key), recentTime, ffgType); ratio = FFMPUtils.getRatioValue(qpe, guid); } @@ -4043,5 +4062,5 @@ public class FFMPResource extends } return dataTimes; } - + } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java index d4df307a8c..f9319bf8a8 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java @@ -32,7 +32,6 @@ import javax.xml.bind.annotation.XmlType; import org.eclipse.core.runtime.SubMonitor; import org.eclipse.core.runtime.jobs.IJobChangeEvent; -import org.eclipse.core.runtime.jobs.Job; import org.eclipse.core.runtime.jobs.JobChangeAdapter; import com.raytheon.uf.common.dataplugin.PluginDataObject; @@ -81,6 +80,7 @@ import com.raytheon.uf.viz.monitor.ffmp.xml.FFMPConfigBasinXML; * Mar 06, 2013 1769 dhladky Changed threading to use count down latch. * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. * Jun 06, 2013 2075 njensen Use new load jobs + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -208,7 +208,6 @@ public class FFMPResourceData extends AbstractRequestableResourceData { } } } - this.domains = defaults; final Date mostRecentTime = availableTimes[availableTimes.length - 1] @@ -216,56 +215,23 @@ public class FFMPResourceData extends AbstractRequestableResourceData { final double configTimeFrame = cfgBasinXML.getTimeFrame(); final Date timeBack = new Date( (long) (mostRecentTime.getTime() - (configTimeFrame * TimeUtil.MILLIS_PER_HOUR))); - final List initialHucs = new ArrayList(); - initialHucs.add(FFMPRecord.ALL); - final String defaultLayer = monitor.getConfig() - .getFFMPConfigData().getLayer(); - if (!defaultLayer.equals(FFMPRecord.ALL)) { - initialHucs.add(defaultLayer); - } + final List onlyAllHuc = new ArrayList(); + onlyAllHuc.add(FFMPRecord.ALL); InitialLoadJob initialJob = new InitialLoadJob(this, timeBack, - mostRecentTime, initialHucs); + mostRecentTime, onlyAllHuc); - // schedule the secondary load to start as soon as the initial - // completes - // secondary load will be the same time period as initial with - // the hucs that the initial job did not do + // schedule the background load to start as soon as the initial + // completes, it will load the other hours back to 24 initialJob.addJobChangeListener(new JobChangeAdapter() { @Override public void done(IJobChangeEvent event) { - Date secondStartTime = timeBack; - List secondaryHucs = FFMPTemplateConfigurationManager - .getInstance().getHucLevels(); - secondaryHucs.removeAll(initialHucs); - - BackgroundLoadJob secondaryJob = new BackgroundLoadJob( - "Secondary FFMP Load", FFMPResourceData.this, - secondStartTime, mostRecentTime, secondaryHucs); - secondaryJob.setPriority(Job.SHORT); - // schedule the tertiary load as soon as the - // secondary completes - // tertiary load will do 24 hours back of the - // same hucs as the initial load - secondaryJob - .addJobChangeListener(new JobChangeAdapter() { - @Override - public void done(IJobChangeEvent event) { - List tertiaryHucs = new ArrayList(); - tertiaryHucs.add(FFMPRecord.ALL); - Date tertiaryStartTime = new Date( - mostRecentTime.getTime() - - (24 * TimeUtil.MILLIS_PER_HOUR)); - BackgroundLoadJob tertiaryJob = new BackgroundLoadJob( - "Tertiary FFMP Load", - FFMPResourceData.this, - tertiaryStartTime, timeBack, - tertiaryHucs); - tertiaryJob - .setPreloadAvailableUris(true); - tertiaryJob.schedule(); - } - }); - secondaryJob.schedule(); + Date backgroundStartTime = new Date(mostRecentTime + .getTime() - (24 * TimeUtil.MILLIS_PER_HOUR)); + BackgroundLoadJob backgroundJob = new BackgroundLoadJob( + "Background FFMP Load", FFMPResourceData.this, + backgroundStartTime, timeBack, onlyAllHuc); + backgroundJob.setPreloadAvailableUris(true); + backgroundJob.schedule(); } }); initialJob.schedule(); @@ -274,7 +240,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData { // background so the first paints of the resource // will be faster List earlyLoadHucs = new ArrayList(); - earlyLoadHucs.addAll(initialHucs); + earlyLoadHucs.addAll(onlyAllHuc); for (String otherHuc : FFMPTemplateConfigurationManager .getInstance().getHucLevels()) { if (!earlyLoadHucs.contains(otherHuc)) { @@ -324,8 +290,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData { .getAvailableUris(siteKey, dataKey, sourceName, standAloneTime); monitor.processUris(sourceURIs, siteKey, sourceName, - standAloneTime, FFMPRecord.ALL, - SubMonitor.convert(null)); + standAloneTime, SubMonitor.convert(null)); } } } @@ -427,10 +392,10 @@ public class FFMPResourceData extends AbstractRequestableResourceData { public void populateRecord(FFMPRecord precord) throws VizException { try { getMonitor().populateFFMPRecord(siteKey, precord, - precord.getSourceName(), huc); + precord.getSourceName()); } catch (Exception e) { throw new VizException("Failed to populate ffmp record " - + precord.getDataURI() + " for huc " + huc); + + precord.getDataURI()); } } diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java index adbce16edc..92be1b7361 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java @@ -59,6 +59,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * Jun 11, 2013 2085 njensen Initial creation + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -208,7 +209,7 @@ public class FFMPRowGenerator implements Runnable { new FFMPTableCellData(rowField, sb.toString(), mouseOverText)); - if (!isWorstCase || huc.equals(FFMPRecord.ALL) + if (huc.equals(FFMPRecord.ALL) || (centeredAggregationKey != null)) { if (!cBasin.getValues().isEmpty()) { @@ -272,8 +273,73 @@ public class FFMPRowGenerator implements Runnable { FIELDS.DIFF, diffValue)); i += 3; } - } else { + } else if (isWorstCase) { trd = getMaxValue(trd, cBasin); + } else { + // general Aggregate HUC processing + ArrayList pfafs = ft.getAggregatePfafs( + cBasin.getPfaf(), siteKey, domain); + + if (!cBasin.getValues().isEmpty()) { + rate = vgBasin.getAverageValue(paintRefTime, + expirationTime); + if (sliderTime > 0.00) { + FFMPTimeWindow window = monitor.getQpeWindow(); + qpeBasin.getAccumAverageValue(pfafs, + window.getAfterTime(), + window.getBeforeTime(), expirationTime, + isRate); + } else { + qpe = 0.0f; + } + } + + trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE, + rate)); + trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE, + qpe)); + + if (qpfBasin != null) { + FFMPTimeWindow window = monitor.getQpfWindow(); + qpf = qpfBasin.getAverageValue(pfafs, + window.getAfterTime(), window.getBeforeTime()); + } + trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF, + qpf)); + + // run over each guidance type + int i = 0; + for (String guidType : guidBasins.keySet()) { + guidance = Float.NaN; + + FFMPTableCellData guidCellData = getGuidanceCellData( + cBasin, domain, guidType, parentBasinPfaf); + if (guidCellData == null) { + // check for forcing even if no data are available + guidance = getForcedAvg(domain, cBasin, guidType); + boolean forced = !guidance.isNaN(); + guidCellData = new FFMPTableCellData( + FIELDS.GUIDANCE, guidance, forced); + } else { + guidance = guidCellData.getValueAsFloat(); + } + + trd.setTableCellData(i + 4, guidCellData); + + float ratioValue = Float.NaN; + float diffValue = Float.NaN; + + // If guidance is NaN then it cannot be > 0 + if (!qpe.isNaN() && (guidance > 0.0f)) { + ratioValue = FFMPUtils.getRatioValue(qpe, guidance); + diffValue = FFMPUtils.getDiffValue(qpe, guidance); + } + trd.setTableCellData(i + 5, new FFMPTableCellData( + FIELDS.RATIO, ratioValue)); + trd.setTableCellData(i + 6, new FFMPTableCellData( + FIELDS.DIFF, diffValue)); + i += 3; + } } trd.setSortCallback(tData); @@ -292,22 +358,46 @@ public class FFMPRowGenerator implements Runnable { if (!isWorstCase || huc.equals(FFMPRecord.ALL) || (centeredAggregationKey != null)) { + + ArrayList pfafs = null; + + if (cBasin.getAggregated()) { + pfafs = ft.getAggregatePfafs(cBasin.getPfaf(), siteKey, + huc); + } + if (rateBasin != null) { - FFMPBasin basin = rateBasin.get(cBasinPfaf); - if (basin != null) { - rate = basin.getValue(paintRefTime); + if (cBasin.getAggregated()) { + rate = rateBasin.getAverageValue(pfafs, + paintRefTime); + } else { + FFMPBasin basin = rateBasin.get(cBasinPfaf); + if (basin != null) { + rate = basin.getValue(paintRefTime); + } } + } trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE, rate)); if (qpeBasin != null) { - FFMPBasin basin = qpeBasin.get(cBasinPfaf); - if (basin != null) { - FFMPTimeWindow window = monitor.getQpeWindow(); - qpe = basin.getAccumValue(window.getAfterTime(), + + FFMPTimeWindow window = monitor.getQpeWindow(); + + if (cBasin.getAggregated()) { + qpe = qpeBasin.getAccumAverageValue(pfafs, + window.getAfterTime(), window.getBeforeTime(), expirationTime, isRate); + } else { + FFMPBasin basin = qpeBasin.get(cBasinPfaf); + if (basin != null) { + qpe = basin.getAccumValue( + window.getAfterTime(), + window.getBeforeTime(), expirationTime, + isRate); + } } } @@ -315,11 +405,19 @@ public class FFMPRowGenerator implements Runnable { qpe)); if (qpfBasin != null) { - FFMPBasin basin = qpfBasin.get(cBasinPfaf); - if (basin != null) { - FFMPTimeWindow window = monitor.getQpfWindow(); - qpf = basin.getAverageValue(window.getAfterTime(), + FFMPTimeWindow window = monitor.getQpfWindow(); + if (cBasin.getAggregated()) { + qpf = qpfBasin.getAverageValue(pfafs, + window.getAfterTime(), window.getBeforeTime()); + } else { + FFMPBasin basin = qpfBasin.get(cBasinPfaf); + if (basin != null) { + + qpf = basin.getAverageValue( + window.getAfterTime(), + window.getBeforeTime()); + } } } @@ -436,13 +534,13 @@ public class FFMPRowGenerator implements Runnable { if (cBasin instanceof FFMPVirtualGageBasin) { if (!pfafs.isEmpty()) { if (virtualBasin != null) { - rate = virtualBasin.get(cBasin.getPfaf()).getValue( - paintRefTime); + + rate = virtualBasin.getMaxValue(pfafs, paintRefTime); if (sliderTime > 0.00) { - qpe = virtualBasin.get(cBasin.getPfaf()).getAccumValue( - monitor.getQpeWindow().getAfterTime(), - monitor.getQpeWindow().getBeforeTime(), + qpe = virtualBasin.getAccumMaxValue(pfafs, monitor + .getQpeWindow().getAfterTime(), monitor + .getQpeWindow().getBeforeTime(), expirationTime, isRate); } else { qpe = 0.0f; @@ -454,10 +552,9 @@ public class FFMPRowGenerator implements Runnable { trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE, qpe)); if (qpfBasin != null) { - qpf = new Float(qpfBasin.get(cBasin.getPfaf()).getMaxValue( - monitor.getQpfWindow().getAfterTime(), - monitor.getQpfWindow().getBeforeTime())) - .floatValue(); + qpf = qpfBasin.getMaxValue(pfafs, monitor.getQpfWindow() + .getAfterTime(), monitor.getQpfWindow() + .getBeforeTime()); } trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF, qpf)); @@ -534,7 +631,7 @@ public class FFMPRowGenerator implements Runnable { if (isWorstCase) { guidance = guidRecords .get(guidType) - .getBasinData(FFMPRecord.ALL) + .getBasinData() .getMaxGuidanceValue( pfafs, resource.getGuidanceInterpolators() @@ -542,11 +639,16 @@ public class FFMPRowGenerator implements Runnable { resource.getGuidSourceExpiration(guidType), cBasin.getPfaf()); } else { - FFMPGuidanceBasin basin = (FFMPGuidanceBasin) guidRecords - .get(guidType).getBasinData(huc) - .get(cBasin.getPfaf()); - guidance = resource.getGuidanceValue(basin, monitor - .getQpeWindow().getBeforeTime(), guidType); + + guidance = guidRecords + .get(guidType) + .getBasinData() + .getAverageGuidanceValue( + pfafs, + resource.getGuidanceInterpolators() + .get(guidType), + resource.getGuidSourceExpiration(guidType)); + } trd.setTableCellData(i + 4, new FFMPTableCellData( @@ -766,17 +868,6 @@ public class FFMPRowGenerator implements Runnable { private FFMPTableCellData getGuidanceCellData(FFMPBasin cBasin, String domain, String guidType, Long parentBasinPfaf) { long cBasinPfaf = cBasin.getPfaf(); - - FFMPBasinData guidBasin = guidBasins.get(guidType); - - FFMPGuidanceBasin ffmpGuidBasin = null; - if (guidBasin != null) { - ffmpGuidBasin = (FFMPGuidanceBasin) guidBasin.get(cBasinPfaf); - } - - if (ffmpGuidBasin == null) { - return null; - } List pfafList = Collections.emptyList(); List forcedPfafs = Collections.emptyList(); boolean forced = false; @@ -806,8 +897,7 @@ public class FFMPRowGenerator implements Runnable { if (FFFGDataMgr.getInstance().isForcingConfigured()) { FFMPBasin parentBasin = cBasin; if (cBasinPfaf != parentBasinPfaf.longValue()) { - parentBasin = baseRec.getBasinData(FFMPRecord.ALL).get( - parentBasinPfaf); + parentBasin = baseRec.getBasinData().get(parentBasinPfaf); } ForceUtilResult forceResult = forceUtil.calculateForcings(domain, ft, parentBasin); @@ -819,20 +909,22 @@ public class FFMPRowGenerator implements Runnable { // Recalculate guidance using the forced value(s) guidance = guidRecords .get(guidType) - .getBasinData(FFMPRecord.ALL) + .getBasinData() .getAverageGuidanceValue(pfafList, resource.getGuidanceInterpolators().get(guidType), guidance, forcedPfafs, resource.getGuidSourceExpiration(guidType)); } else { - if (ffmpGuidBasin != null) { - guidance = resource.getGuidanceValue(ffmpGuidBasin, - paintRefTime, guidType); - if (guidance < 0.0f) { - guidance = Float.NaN; - } + FFMPGuidanceBasin ffmpGuidBasin = (FFMPGuidanceBasin) guidRecords + .get(guidType).getBasinData().get(cBasinPfaf); + guidance = resource.getGuidanceValue(ffmpGuidBasin, paintRefTime, + guidType); + + if (guidance < 0.0f) { + guidance = Float.NaN; } + } return new FFMPTableCellData(FIELDS.GUIDANCE, guidance, forced); diff --git a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java index 02ae95e599..3d7e608248 100644 --- a/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java +++ b/cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java @@ -22,7 +22,6 @@ package com.raytheon.uf.viz.monitor.ffmp.ui.thread; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -31,7 +30,6 @@ import java.util.Set; import org.eclipse.core.runtime.SubMonitor; import org.eclipse.core.runtime.jobs.Job; -import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord; import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager; import com.raytheon.uf.common.monitor.xml.FFMPRunXML; import com.raytheon.uf.common.monitor.xml.ProductRunXML; @@ -54,6 +52,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.rsc.FFMPResourceData; * ------------ ---------- ----------- -------------------------- * Jun 04, 2013 2075 njensen Initial creation * Jun 07, 2013 2075 njensen Added progress monitoring + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -81,8 +80,6 @@ public abstract class AbstractLoadJob extends Job { protected FFMPMonitor ffmpMonitor; - protected List hucsToLoad = null; - /** * Constructor * @@ -105,28 +102,13 @@ public abstract class AbstractLoadJob extends Job { this.resourceData = resourceData; this.startTime = timeBack; this.endTime = mostRecentTime; - this.hucsToLoad = hucsToLoad; // configure FFMP - this.hucsToLoad.remove(FFMPRecord.VIRTUAL); FFMPRunXML runXML = FFMPRunConfigurationManager.getInstance() .getRunner(resourceData.wfo); this.productRun = runXML.getProduct(resourceData.siteKey); this.product = resourceData.getProduct(); this.ffmpMonitor = FFMPMonitor.getInstance(); - - // just for debugging/logging - StringBuilder sb = new StringBuilder(); - sb.append(name); - sb.append(" hucs to load: "); - Iterator itr = this.hucsToLoad.iterator(); - while (itr.hasNext()) { - sb.append(itr.next()); - if (itr.hasNext()) { - sb.append(", "); - } - } - System.out.println(sb.toString()); } /** @@ -165,10 +147,8 @@ public abstract class AbstractLoadJob extends Job { } } if (rateURI != null) { - for (String phuc : hucsToLoad) { - ffmpMonitor.processUri(rateURI, resourceData.siteKey, - product.getRate(), startTime, phuc); - } + ffmpMonitor.processUri(rateURI, resourceData.siteKey, + product.getRate(), startTime); } } @@ -190,12 +170,10 @@ public abstract class AbstractLoadJob extends Job { protected void doQpe(NavigableMap> qpeURIs, SubMonitor smonitor) { if (!qpeURIs.isEmpty()) { - smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR); - for (String phuc : hucsToLoad) { - ffmpMonitor.processUris(qpeURIs, resourceData.siteKey, - product.getQpe(), startTime, phuc, - smonitor.newChild(PROGRESS_FACTOR)); - } + smonitor.beginTask(null, PROGRESS_FACTOR); + ffmpMonitor.processUris(qpeURIs, resourceData.siteKey, + product.getQpe(), startTime, + smonitor.newChild(PROGRESS_FACTOR)); } } @@ -246,12 +224,11 @@ public abstract class AbstractLoadJob extends Job { // Use this method of QPF data retrieval if you don't have cache // files if (!qpfURIs.isEmpty()) { - smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR); - for (String phuc : hucsToLoad) { - ffmpMonitor.processUris(qpfURIs, resourceData.siteKey, - productQpf, startTime, phuc, - smonitor.newChild(PROGRESS_FACTOR)); - } + smonitor.beginTask(null, PROGRESS_FACTOR); + + ffmpMonitor.processUris(qpfURIs, resourceData.siteKey, productQpf, + startTime, smonitor.newChild(PROGRESS_FACTOR)); + } } @@ -264,7 +241,7 @@ public abstract class AbstractLoadJob extends Job { product.getVirtual(), startTime); if (!virtualURIs.isEmpty()) { ffmpMonitor.processUris(virtualURIs, resourceData.siteKey, - product.getVirtual(), startTime, FFMPRecord.ALL, smonitor); + product.getVirtual(), startTime, smonitor); } } @@ -303,7 +280,7 @@ public abstract class AbstractLoadJob extends Job { if (iguidURIs != null && !iguidURIs.isEmpty()) { ffmpMonitor.processUris(iguidURIs, resourceData.siteKey, guidSource.getSourceName(), startTime, - FFMPRecord.ALL, + smonitor.newChild(PROGRESS_FACTOR / subWork)); } } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java index 46c3c93e40..f4aacea68b 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java @@ -21,7 +21,6 @@ package com.raytheon.uf.common.dataplugin.ffmp; **/ import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import com.raytheon.uf.common.serialization.ISerializableObject; @@ -37,6 +36,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * 01/27/13 1478 D. Hladky Created to reduce memory and disk read/writes for FFMP + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -52,8 +52,6 @@ public class FFMPAggregateRecord implements ISerializableObject { @DynamicSerializeElement private String sourceSiteDataKey; - - private static final long serialVersionUID = 767745643535471L; /** * Aggregate Record implementation for FFMP @@ -63,8 +61,16 @@ public class FFMPAggregateRecord implements ISerializableObject { } @DynamicSerializeElement - private HashMap basinsMap = new HashMap(); + private FFMPBasinData basins; + public FFMPBasinData getBasins() { + return basins; + } + + public void setBasins(FFMPBasinData basins) { + this.basins = basins; + } + @DynamicSerializeElement private List times = new ArrayList(); @@ -83,34 +89,6 @@ public class FFMPAggregateRecord implements ISerializableObject { public List getTimes() { return times; } - - public void setBasinsMap(HashMap basinsMap) { - this.basinsMap = basinsMap; - } - - public HashMap getBasinsMap() { - return basinsMap; - } - - /** - * Add a basin Data aggregate object - * @param cacheData - */ - public void addBasinData(FFMPBasinData basinData) { - basinsMap.put(basinData.getHucLevel(), basinData); - } - - /** - * Gets the BasinData object - * @param huc - * @return - */ - public FFMPBasinData getBasinData(String huc) { - if (basinsMap.containsKey(huc)) { - return basinsMap.get(huc); - } - return null; - } /** * WFO designator diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java index 715d6ea50f..4f0ebe8216 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java @@ -31,7 +31,6 @@ import javax.persistence.Transient; import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; -import com.raytheon.uf.common.time.util.ImmutableDate; /** * FFMP basin/aggregated value holder @@ -48,6 +47,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate; * from aggregate records and delayed * TreeMap creation to the tertiary loader. * Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP. + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java index d158cd791b..fa047dbc0d 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java @@ -51,6 +51,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * 01/27/13 1569 D. Hladky Added support for write of aggregate record cache * 04/16/13 1912 bsteffen Initial bulk hdf5 access for ffmp * 05/09/13 1919 mpduff Use parent pfaf instead of lookupId. + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -59,7 +60,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; */ @DynamicSerialize public class FFMPBasinData implements ISerializableObject { - private static final long serialVersionUID = 8162247989509750715L; public static final double GUIDANCE_MISSING = -999999.0; @@ -80,15 +80,6 @@ public class FFMPBasinData implements ISerializableObject { */ private final Map orderedBasinsCache = new HashMap(); - /** - * Public one arg constructor - * - * @param huc_level - */ - public FFMPBasinData(String hucLevel) { - setHucLevel(hucLevel); - } - /** * No arg hibernate constructor */ @@ -172,6 +163,8 @@ public class FFMPBasinData implements ISerializableObject { * Extracts the average value for an aggregation of basins * * @param pfaf_ids + * @param startDate + * @param finishDate * @return */ public float getAverageValue(ArrayList pfaf_ids, Date beforeDate, @@ -191,6 +184,55 @@ public class FFMPBasinData implements ISerializableObject { return tvalue; } + /** + * Extracts the average value for an aggregation of basins + * + * @param pfaf_ids + * @param exact + * date + * @return + */ + public float getAverageValue(ArrayList pfaf_ids, Date date) { + + float tvalue = 0.0f; + int i = 0; + for (Long pfaf : pfaf_ids) { + FFMPBasin basin = getBasins().get(pfaf); + if (basin != null) { + tvalue += basin.getValue(date); + i++; + } + } + tvalue = tvalue / i; + + return tvalue; + } + + /** + * Extracts the average value for an aggregation of basins + * + * @param pfaf_ids + * @param date + * @param expirationTime + * @return + */ + public float getAverageValue(ArrayList pfaf_ids, Date date, + long epirationTime) { + + float tvalue = 0.0f; + int i = 0; + for (Long pfaf : pfaf_ids) { + FFMPBasin basin = getBasins().get(pfaf); + if (basin != null) { + tvalue += basin.getAverageValue(date, epirationTime); + i++; + } + } + tvalue = tvalue / i; + + return tvalue; + } + /** * Extracts the average value for an aggregation of basins with areal * weighting @@ -229,8 +271,8 @@ public class FFMPBasinData implements ISerializableObject { * @param pfaf_ids * @return */ - public float getAccumAverageValue(ArrayList pfaf_ids, - Date beforeDate, Date afterDate, long expirationTime, boolean rate) { + public float getAccumAverageValue(List pfaf_ids, Date beforeDate, + Date afterDate, long expirationTime, boolean rate) { float tvalue = 0.0f; int i = 0; @@ -534,6 +576,34 @@ public class FFMPBasinData implements ISerializableObject { return values; } + /** + * Gets the average guidance value for an aggregate basin + * + * @param pfaf_ids + * @param interpolation + * @param expiration + * @return + */ + public Float getAverageGuidanceValue(List pfaf_ids, + FFMPGuidanceInterpolation interpolation, long expiration) { + + float tvalue = 0.0f; + int i = 0; + + List vals = getGuidanceValues(pfaf_ids, interpolation, + expiration); + if (vals != null) { + for (Float val : vals) { + tvalue += val; + i++; + } + } else { + return null; + } + + return tvalue / i; + } + /** * used for max ratio and diff calcs * diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java index 812957e6b4..e357c04e04 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java @@ -24,8 +24,6 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.SOURCE_TYPE; import com.raytheon.uf.common.monitor.xml.SourceXML; @@ -47,6 +45,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException * 01/27/13 1478 D. Hladky Re-worked to help with memory size and NAS read write stress * Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -59,10 +58,10 @@ public class FFMPDataContainer { private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(FFMPDataContainer.class); - private final Map basinDataMap = new ConcurrentHashMap();// DR + private FFMPBasinData basins; private String sourceName = null; - + private boolean isPurged = false; public FFMPDataContainer() { @@ -71,40 +70,41 @@ public class FFMPDataContainer { /** * Usual constructor + * * @param sourceName */ public FFMPDataContainer(String sourceName) { this.sourceName = sourceName; - basinDataMap.put(FFMPRecord.ALL, new FFMPBasinData(FFMPRecord.ALL)); - // System.out.println("Creating source: " + sourceName); + basins = new FFMPBasinData(); } /** * special constuctor + * * @param sourceName * @param hucs */ public FFMPDataContainer(String sourceName, ArrayList hucs) { // System.out.println("Creating source with hucs: " + sourceName); this.sourceName = sourceName; - for (String huc : hucs) { - basinDataMap.put(huc, new FFMPBasinData(huc)); - } + basins = new FFMPBasinData(); } + /** * new container first time read in from cache + * * @param sourceName * @param hucs * @param record */ - public FFMPDataContainer(String sourceName, ArrayList hucs, FFMPAggregateRecord record) { + public FFMPDataContainer(String sourceName, FFMPAggregateRecord record) { // System.out.println("Creating source with hucs: " + sourceName); this.sourceName = sourceName; - for (String huc : hucs) { - FFMPBasinData basinData = record.getBasinData(huc); - basinData.populate(record.getTimes()); - basinDataMap.put(huc, basinData); - } + + FFMPBasinData basinData = record.getBasins(); + basinData.populate(record.getTimes()); + basins = basinData; + } /** @@ -114,7 +114,7 @@ public class FFMPDataContainer { * @param newBasinData */ public void addFFMPEntry(Date date, SourceXML source, - FFMPBasinData newBasinData, String huc, String siteKey) { + FFMPBasinData newBasinData, String siteKey) { boolean guid = false; @@ -122,12 +122,12 @@ public class FFMPDataContainer { guid = true; } - FFMPBasinData currBasinData = getBasinData(huc); + FFMPBasinData currBasinData = getBasinData(); synchronized (currBasinData) { if (currBasinData == null) { - setBasinData(huc, newBasinData); + setBasinData(newBasinData); } else { for (Long key : newBasinData.getBasins().keySet()) { @@ -158,7 +158,7 @@ public class FFMPDataContainer { val); } - //currBasinData.put(key, basin); + // currBasinData.put(key, basin); syncPut(currBasinData, key, basin); } else { @@ -243,7 +243,7 @@ public class FFMPDataContainer { } basin.setValue(date, val); - //currBasinData.put(key, basin); + // currBasinData.put(key, basin); syncPut(currBasinData, key, basin); } else { @@ -283,10 +283,9 @@ public class FFMPDataContainer { public boolean containsKey(Date date) { boolean contains = false; - if (getBasinData(FFMPRecord.ALL) != null) { + if (getBasinData() != null) { - Map basins = getBasinData(FFMPRecord.ALL) - .getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -309,7 +308,7 @@ public class FFMPDataContainer { */ public boolean containsKey(String sourceName) { boolean contains = false; - Map basins = getBasinData(FFMPRecord.ALL).getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -334,18 +333,14 @@ public class FFMPDataContainer { * * @return */ - public FFMPBasinData getBasinData(String huc) { - if (basinDataMap.containsKey(huc)) { - return basinDataMap.get(huc); + public FFMPBasinData getBasinData() { + if (basins != null) { + return basins; } else { return null; } } - public Set getKeys() { - return basinDataMap.keySet(); - } - /** * Get the maximum value in the monitored area. * @@ -359,9 +354,8 @@ public class FFMPDataContainer { public double getMaxValue(ArrayList pfafs, Date backDate, Date currDate, long expirationTime, boolean rate) { - double val = getBasinData(FFMPRecord.ALL).getAccumMaxValue(pfafs, - currDate, - backDate, expirationTime, rate); + double val = getBasinData().getAccumMaxValue(pfafs, currDate, backDate, + expirationTime, rate); return val; } @@ -374,8 +368,7 @@ public class FFMPDataContainer { public Date getNewest() { try { - Map basins = getBasinData(FFMPRecord.ALL) - .getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -402,8 +395,7 @@ public class FFMPDataContainer { */ public Date getOldest() { try { - Map basins = getBasinData(FFMPRecord.ALL) - .getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -431,8 +423,7 @@ public class FFMPDataContainer { public List getOrderedTimes(Date barrierTime) { ArrayList orderedTimes = new ArrayList(); try { - Map basins = getBasinData(FFMPRecord.ALL) - .getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -453,16 +444,16 @@ public class FFMPDataContainer { return null; } - + /** * Gets the list of times for serialization + * * @return */ public List getOrderedTimes() { ArrayList orderedTimes = new ArrayList(); try { - Map basins = getBasinData(FFMPRecord.ALL) - .getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -484,6 +475,7 @@ public class FFMPDataContainer { /** * Gets the source name for this Data Container + * * @return */ public String getSourceName() { @@ -492,13 +484,13 @@ public class FFMPDataContainer { /** * Clean up old junk + * * @param backDate */ public void purge(Date backDate) { - statusHandler.handle(Priority.INFO, "Purging "+getSourceName()+" Container back to: "+backDate); - for (String huc : basinDataMap.keySet()) { - getBasinData(huc).purgeData(backDate); - } + statusHandler.handle(Priority.INFO, "Purging " + getSourceName() + + " Container back to: " + backDate); + getBasinData().purgeData(backDate); setPurged(true); } @@ -511,27 +503,25 @@ public class FFMPDataContainer { // create a record from the cache record FFMPRecord record = new FFMPRecord(cacheRecord); - - for (Entry dentry : record.getBasinsMap() - .entrySet()) { - for (Entry entry : dentry.getValue().getBasins() - .entrySet()) { - FFMPBasin basin = entry.getValue(); - if (basin != null) { - if (basin instanceof FFMPGuidanceBasin) { - FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin; - gbasin.getGuidValues().putAll( - ((FFMPGuidanceBasin) entry.getValue()) - .getGuidValues()); - } else { - basin.getValues().putAll(entry.getValue().getValues()); - } + + FFMPBasinData dentry = record.getBasinData(); + + for (Entry entry : dentry.getBasins().entrySet()) { + FFMPBasin basin = entry.getValue(); + if (basin != null) { + if (basin instanceof FFMPGuidanceBasin) { + FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin; + gbasin.getGuidValues().putAll( + ((FFMPGuidanceBasin) entry.getValue()) + .getGuidValues()); } else { - syncPut(getBasinData(dentry.getKey()), entry.getKey(), - entry.getValue()); + basin.getValues().putAll(entry.getValue().getValues()); } + } else { + syncPut(getBasinData(), entry.getKey(), entry.getValue()); } } + } /** @@ -539,8 +529,8 @@ public class FFMPDataContainer { * * @param basins */ - public void setBasinData(String huc, FFMPBasinData fftiData) { - basinDataMap.put(huc, fftiData); + public void setBasinData(FFMPBasinData basins) { + this.basins = basins; } public void setSourceName(String sourceName) { @@ -555,7 +545,7 @@ public class FFMPDataContainer { */ public int size() { - Map basins = getBasinData(FFMPRecord.ALL).getBasins(); + Map basins = getBasinData().getBasins(); synchronized (basins) { for (Entry entry : basins.entrySet()) { @@ -569,34 +559,27 @@ public class FFMPDataContainer { } return 0; } - + /** - * DR 15471 lock put() to avoid ConcurrentModificationException - */ - - private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value){ - if(fbd==null || key==null) - return; - - Map basins = fbd.getBasins(); - if(basins == null) - return; - - synchronized (basins) { - basins.put(key, value); - } - } - - /** - * Gets the basin data map - * @return + * DR 15471 lock put() to avoid ConcurrentModificationException */ - public Map getBasinMap() { - return basinDataMap; + + private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value) { + if (fbd == null || key == null) + return; + + Map basins = fbd.getBasins(); + if (basins == null) + return; + + synchronized (basins) { + basins.put(key, value); + } } /** * Sets whether this container has been purged or not + * * @param isPurged */ public void setPurged(boolean isPurged) { @@ -605,6 +588,7 @@ public class FFMPDataContainer { /** * Has this container been purged? + * * @return */ public boolean isPurged() { diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java index 3979d310c9..139a2b9826 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java @@ -24,7 +24,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.lang.ref.WeakReference; import java.util.Date; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -89,6 +88,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate; * Apr 18, 2013 1919 dhladky Added method for VGB loading * May 07, 2013 1869 bsteffen Remove dataURI column from * PluginDataObject. + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -142,23 +142,25 @@ public class FFMPRecord extends PersistablePluginDataObject private String siteKey; @Transient - private HashMap basinsMap = new HashMap(); + private FFMPBasinData basins = new FFMPBasinData();; @Transient private int expiration = 0; @Transient private boolean isRate = false; - + protected static ConcurrentMap> cacheTimes = new ConcurrentHashMap>(); private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(FFMPRecord.class); - + /** ALL HUC LEVEL **/ public static final String ALL = "ALL"; + /** COUNTY HUC LEVEL **/ public static final String COUNTY = "COUNTY"; + /** VIRTUAL HUC LEVEL **/ public static final String VIRTUAL = "VIRTUAL"; @@ -359,8 +361,8 @@ public class FFMPRecord extends PersistablePluginDataObject * @param hucName */ - public void setBasinData(FFMPBasinData basins, String hucName) { - basinsMap.put(hucName, basins); + public void setBasinData(FFMPBasinData basins) { + this.basins = basins; } /** @@ -369,24 +371,10 @@ public class FFMPRecord extends PersistablePluginDataObject * @param basins * @param hucName */ - public FFMPBasinData getBasinData(String hucName) { - FFMPBasinData basins = basinsMap.get(hucName); - if (basins == null) { - basins = new FFMPBasinData(hucName); - basinsMap.put(hucName, basins); - } + public FFMPBasinData getBasinData() { return basins; } - /** - * Gets the map if you need it - * - * @return - */ - public HashMap getBasinsMap() { - return basinsMap; - } - /** * Gets the Hash out of the datastore by HUC * @@ -394,41 +382,36 @@ public class FFMPRecord extends PersistablePluginDataObject * @param huc */ public void retrieveMapFromDataStore(File datastoreFile, String uri, - FFMPTemplates template, String huc, Date date, String sourceName) + FFMPTemplates template, Date date, String sourceName) throws Exception { - FFMPBasinData fbd = getBasinData(huc); + FFMPBasinData fbd = getBasinData(); ImmutableDate idate = getCacheDate(date); - - boolean aggregate = true; - - if (huc.equals(ALL)) { - aggregate = false; - } + boolean aggregate = false; for (DomainXML domain : template.getDomains()) { LinkedHashMap map = template.getMap(getSiteKey(), - domain.getCwa(), huc); + domain.getCwa(), FFMPRecord.ALL); if (map != null && !map.isEmpty()) { fbd.addBasins(datastoreFile, uri, getSiteKey(), - domain.getCwa(), huc, sourceName, idate, map.keySet(), - aggregate); + domain.getCwa(), FFMPRecord.ALL, sourceName, idate, + map.keySet(), aggregate); } } } - public void retrieveMapFromDataStore(FFMPTemplates template, String huc) + public void retrieveMapFromDataStore(FFMPTemplates template) throws Exception { retrieveMapFromDataStore(getDataStoreFile(), getDataURI(), template, - huc, getDataTime().getRefTime(), getSourceName()); + getDataTime().getRefTime(), getSourceName()); } - public void retrieveVirtualMapFromDataStore(FFMPTemplates template, - String huc) throws Exception { - retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(), template, - getDataTime().getRefTime(), getSourceName()); + public void retrieveVirtualMapFromDataStore(FFMPTemplates template) + throws Exception { + retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(), + template, getDataTime().getRefTime(), getSourceName()); } private File getDataStoreFile() { @@ -523,7 +506,7 @@ public class FFMPRecord extends PersistablePluginDataObject public void retrieveVirtualMapFromDataStore(File datastoreFile, String uri, FFMPTemplates template, Date date, String sourceName) throws StorageException, FileNotFoundException { - FFMPBasinData fbd = getBasinData(ALL); + FFMPBasinData fbd = getBasinData(); String key = getDataKey(); ImmutableDate idate = getCacheDate(date); @@ -537,8 +520,7 @@ public class FFMPRecord extends PersistablePluginDataObject if (size > 0) { fbd.addVirtualBasins(datastoreFile, uri, key, - domain.getCwa(), idate, - lids.values()); + domain.getCwa(), idate, lids.values()); } } } @@ -569,10 +551,9 @@ public class FFMPRecord extends PersistablePluginDataObject public String toString() { StringBuffer sb = new StringBuffer(); sb.append("\n dataURI: " + getDataURI() + "\n"); - if (basinsMap != null) { - for (String key : basinsMap.keySet()) { - sb.append(key + " : " + basinsMap.get(key).getBasins().size() - + "\n"); + if (basins != null) { + for (Long key : basins.getBasins().keySet()) { + sb.append(key + " : " + basins.get(key).getValue() + "\n"); } } @@ -635,10 +616,7 @@ public class FFMPRecord extends PersistablePluginDataObject * @param date */ public void purgeData(Date date) { - - for (FFMPBasinData basinData : getBasinsMap().values()) { - basinData.purgeData(date); - } + basins.purgeData(date); } public void setSiteKey(String siteKey) { @@ -648,21 +626,20 @@ public class FFMPRecord extends PersistablePluginDataObject public String getSiteKey() { return siteKey; } - + /** * Get the fully populated aggregate record + * * @return */ public FFMPAggregateRecord getAggregateRecord() { FFMPAggregateRecord fdcr = new FFMPAggregateRecord(); - - for (FFMPBasinData basinData: basinsMap.values()) { - fdcr.addBasinData(basinData); - } - + + fdcr.setBasins(basins); + return fdcr; } - + /** * Creates and populates a version of this record from an aggregate record * @@ -671,14 +648,11 @@ public class FFMPRecord extends PersistablePluginDataObject public FFMPRecord(FFMPAggregateRecord fdcr) { List times = fdcr.getTimes(); - - for (FFMPBasinData basinData: fdcr.getBasinsMap().values()) { - // Keep in mind times can be null, Guidance basins are like that - basinData.populate(times); - setBasinData(basinData, basinData.getHucLevel()); - } + FFMPBasinData fdcrBasins = fdcr.getBasins(); + fdcrBasins.populate(times); + setBasinData(fdcrBasins); } - + /** * Gets and maintains the list of times. This will lesson memory consumption * because it means all FFMPBasin TreeMap date keys reference back to this @@ -691,7 +665,7 @@ public class FFMPRecord extends PersistablePluginDataObject WeakReference idate = cacheTimes.get(date.getTime()); ImmutableDate myDate = null; - + if (idate != null) { myDate = idate.get(); } @@ -701,22 +675,19 @@ public class FFMPRecord extends PersistablePluginDataObject myDate = new ImmutableDate(time); idate = new WeakReference(myDate); cacheTimes.putIfAbsent(time, idate); - } + } return myDate; } - - + /** * Populate data from the cache files + * * @param basins * @param hucName */ - public void populate(FFMPBasinData basins, String hucName) { - - setBasinData(basins, hucName); - //System.out.println("Adding Whole Object Cache Data: "+hucName+" "+getSourceName()); - + public void populate(FFMPBasinData basins) { + setBasinData(basins); } @Override diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java index 9d3d81a557..f67ae97e9e 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java @@ -88,6 +88,7 @@ import com.vividsolutions.jts.io.WKBReader; * 03/18/13 1817 D. Hladky Fixed issue with BOX where only 1 HUC was showing up. * 04/15/13 1902 M. Duff Generic List * 06/10/13 2085 njensen Use countyMap for efficiency + * 07/15/13 2184 dhladky Remove all HUC's for storage except ALL * * * @author dhladky @@ -1310,6 +1311,25 @@ public class FFMPTemplates { return map; } + + /** + * Find the list of pfafs for this HUC level + * @param siteKey + * @param huc + * @param domains + * @return + */ + public synchronized List getHucKeyList(String siteKey, String huc, List domains) { + + Set keys = new HashSet(); + + for (DomainXML domain: domains) { + LinkedHashMap map = getMap(siteKey, domain.getCwa(), huc); + keys.addAll(map.keySet()); + } + + return new ArrayList(keys); + } /** * Gets the template config manager @@ -1663,23 +1683,28 @@ public class FFMPTemplates { } HashMap>> virtualMap = virtualGageBasinsInParentPfaf .get(dataKey); + ArrayList result = new ArrayList(); for (DomainXML domain : domains) { HashMap> map = virtualMap .get(domain.getCwa()); if (map != null) { + ArrayList list = map.get(pfaf); + if (list != null && !list.isEmpty()) { - ArrayList result = new ArrayList(); - for (FFMPVirtualGageBasinMetaData md : list) - result.add(md.getLookupId()); - return result; + + for (FFMPVirtualGageBasinMetaData md : list) { + if (!result.contains(md.getLookupId())) { + result.add(md.getLookupId()); + } + } } } } - return new ArrayList(); + return result; } @@ -2423,6 +2448,8 @@ public class FFMPTemplates { HashMap>> virtualMap = vgbsInCounty .get(dataKey); + ArrayList result = new ArrayList(); + for (DomainXML domain : domains) { HashMap> map = virtualMap @@ -2432,16 +2459,17 @@ public class FFMPTemplates { .get(stateCommaCnty.trim().toUpperCase()); if (list != null && !list.isEmpty()) { - ArrayList result = new ArrayList(); - for (FFMPVirtualGageBasinMetaData md : list) { - result.add(md.getLookupId()); + for (FFMPVirtualGageBasinMetaData md : list) { + if (!result.contains(md.getLookupId())) { + result.add(md.getLookupId()); + } } - return result; } } } - return new ArrayList(); + + return result; } } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java index d8796e2f97..ae02f0e7ab 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java @@ -53,6 +53,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao; * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- * 07/01/09 2521 dhladky Initial Creation + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -103,63 +104,36 @@ public class FFMPDao extends PluginDao { // ignore data outside of domain if (vmap.size() > 0) { - for (String key : record.getBasinsMap().keySet()) { - FFMPBasinData fbd = record.getBasinData(key); - LinkedHashMap map = template.getMap( - record.getSiteKey(), domain.getCwa(), key); - int size = 0; - if (key.equals(FFMPRecord.ALL)) { - for (Entry entry : vmap - .entrySet()) { - if (entry.getValue() != null) { - size++; - } - } - } else { - for (Long pfaf : map.keySet()) { - ArrayList vgbpfafs = template - .getVGBsInAggregate(pfaf, - record.getSiteKey(), key); - if (vgbpfafs != null) { - size++; - } - } + FFMPBasinData fbd = record.getBasinData(); + int size = 0; + + for (Entry entry : vmap + .entrySet()) { + if (entry.getValue() != null) { + size++; } - - float[] dataRec = new float[size]; - int i = 0; - - if (key.equals(FFMPRecord.ALL)) { - for (Entry entry : vmap - .entrySet()) { - if (entry.getValue() != null) { - FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd - .get(entry.getValue().getLookupId()); - dataRec[i] = bd.getValue(); - i++; - } - } - } else { - for (Long pfaf : map.keySet()) { - ArrayList vgbpfafs = template - .getVGBsInAggregate(pfaf, - record.getSiteKey(), key); - if (vgbpfafs != null) { - FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd - .get(pfaf); - dataRec[i] = bd.getValue(); - i++; - } - } - } - - // NAME | GROUP | array |Dimension | dimensions - IDataRecord rec = new FloatDataRecord(key, - record.getDataURI() + "/" + domain.getCwa(), - dataRec, 1, new long[] { size }); - dataStore.addDataRecord(rec); } + + float[] dataRec = new float[size]; + int i = 0; + + for (Entry entry : vmap + .entrySet()) { + if (entry.getValue() != null) { + FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd + .get(entry.getValue().getLookupId()); + dataRec[i] = bd.getValue(); + i++; + } + } + + // NAME | GROUP | array |Dimension | dimensions + IDataRecord rec = new FloatDataRecord(FFMPRecord.ALL, + record.getDataURI() + "/" + domain.getCwa(), + dataRec, 1, new long[] { size }); + dataStore.addDataRecord(rec); + } else { statusHandler.handle(Priority.DEBUG, "No VGB's in domain: " + domain.getCwa()); @@ -170,44 +144,40 @@ public class FFMPDao extends PluginDao { else { - for (String huc : record.getBasinsMap().keySet()) { + if (record.getBasinData() != null) { - if (record.getBasinData(huc) != null) { + for (DomainXML domain : template.getDomains()) { - for (DomainXML domain : template.getDomains()) { + LinkedHashMap map = template.getMap( + record.getSiteKey(), domain.getCwa(), + FFMPRecord.ALL); + FFMPBasinData fbd = record.getBasinData(); + // ignore data outside domain + if (map.size() > 0 && fbd.getBasins().size() > 0) { + int size = map.size(); - LinkedHashMap map = template.getMap( - record.getSiteKey(), domain.getCwa(), huc); - FFMPBasinData fbd = record.getBasinData(huc); - // ignore data outside domain - if (map.size() > 0 && fbd.getBasins().size() > 0) { - int size = map.size(); - - float[] dataRec = new float[size]; - int i = 0; - // write individual basins, use template, preserves - // ordering - for (Long pfaf : map.keySet()) { - FFMPBasin bd = fbd.get(pfaf); - if (bd != null) { - dataRec[i] = bd.getValue(); - i++; - } + float[] dataRec = new float[size]; + int i = 0; + // write individual basins, use template, preserves + // ordering + for (Long pfaf : map.keySet()) { + FFMPBasin bd = fbd.get(pfaf); + if (bd != null) { + dataRec[i] = bd.getValue(); + i++; } - // NAME | GROUP | array |Dimension | dimensions - if (i > 0) { - IDataRecord rec = new FloatDataRecord(huc, - record.getDataURI() + "/" - + domain.getCwa(), dataRec, 1, - new long[] { size }); - dataStore.addDataRecord(rec); - } - } else { - statusHandler.handle( - Priority.DEBUG, - "Data outside of domain: " - + domain.getCwa()); } + // NAME | GROUP | array |Dimension | dimensions + if (i > 0) { + IDataRecord rec = new FloatDataRecord( + FFMPRecord.ALL, record.getDataURI() + "/" + + domain.getCwa(), dataRec, 1, + new long[] { size }); + dataStore.addDataRecord(rec); + } + } else { + statusHandler.handle(Priority.DEBUG, + "Data outside of domain: " + domain.getCwa()); } } } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java index c818aacaad..b2417c5d8b 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java @@ -64,6 +64,7 @@ import com.vividsolutions.jts.geom.Geometry; * ------------ ---------- ----------- -------------------------- * Jan 24, 2013 1552 mpduff Initial creation * Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @@ -126,8 +127,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory { for (Map.Entry es : map.entrySet()) { FFMPRecord rec = (FFMPRecord) es.getValue(); try { - rec.retrieveMapFromDataStore(templates, (String) request - .getIdentifiers().get(HUC)); + rec.retrieveMapFromDataStore(templates); } catch (Exception e) { throw new DataRetrievalException( "Failed to retrieve the IDataRecord for PluginDataObject: " @@ -204,7 +204,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory { String siteKey = (String) request.getIdentifiers().get(SITE_KEY); String cwa = (String) request.getIdentifiers().get(WFO); - FFMPBasinData basinData = rec.getBasinData(huc); + FFMPBasinData basinData = rec.getBasinData(); Map basinDataMap = basinData.getBasins(); diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java index 3a2c361233..a4f9d6077c 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java @@ -32,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.regex.Pattern; -import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage; +import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage; import com.raytheon.edex.plugin.radar.dao.RadarStationDao; import com.raytheon.edex.urifilter.URIFilter; import com.raytheon.edex.urifilter.URIGenerateMessage; @@ -123,6 +123,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff; * 02/25/13 1660 D. Hladky Redesigned data flow for FFTI in order to have only one mosaic piece in memory at a time. * 03/13/13 1478 D. Hladky non-FFTI mosaic containers weren't getting ejected. Made it so that they are ejected after processing as well. * 03/22/13 1803 D. Hladky Fixed broken performance logging for ffmp. + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @author dhladky @@ -131,9 +132,10 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff; public class FFMPGenerator extends CompositeProductGenerator implements MonitorConfigListener { - + private static final transient IUFStatusHandler statusHandler = UFStatus .getHandler(FFMPGenerator.class); + /** * Public constructor for FFMPGenerator * @@ -152,13 +154,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements private static final String productType = "ffmp"; /** - * The thought was this will eventually be dynamic when front end can support it. + * The thought was this will eventually be dynamic when front end can + * support it. */ public static final int SOURCE_CACHE_TIME = 6; /** - * The thought was this will eventually be dynamic, static in AWIPS I. - * This is the time back limit for Flash Flood Guidance sources + * The thought was this will eventually be dynamic, static in AWIPS I. This + * is the time back limit for Flash Flood Guidance sources */ public static final int FFG_SOURCE_CACHE_TIME = 24; @@ -388,7 +391,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements statusHandler.handle(Priority.PROBLEM, "Couldn't create FFMP Filter.." + " primary Domain: " + domain.getCwa() - + " this RUNNER is not a viable FFMP config.", e); + + " this RUNNER is not a viable FFMP config.", + e); } } @@ -440,7 +444,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements @Override public void generateProduct(URIGenerateMessage genMessage) { - if (loaded) { + if (loaded) { try { long time = System.currentTimeMillis(); this.config = new FFMPConfig( @@ -477,7 +481,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } } catch (InterruptedException e) { - statusHandler.handle(Priority.ERROR, "Process thread had been interupted!", e); + statusHandler.handle(Priority.ERROR, + "Process thread had been interupted!", e); } } @@ -554,6 +559,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements private class ProcessProduct implements Runnable { SourceXML ffmpProduct; + FFMPGenerator generator; @Override @@ -592,7 +598,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements HashMap dataHash = config.getSourceData(ffmpProduct .getSourceName()); - + FFMPRunXML runner = getRunConfig().getRunner(config.getCWA()); // process all of the dataKeys for this source @@ -684,19 +690,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements ffmpRec, template); ffmpRec = ffmp.processFFMP(ffmpProduct); ffmpRec.constructDataURI(); - + if (ffmpRec != null) { - + persistRecord(ffmpRec); processDataContainer(ffmpRec, siteKey); - // Now that we have the data container, + // Now that we have the data container, // we can process FFTI for this piece of the mosaic if (ffmp.isFFTI()) { - + fftiDone = false; FFTISourceXML fftiSource = ffmp.getFFTISource(); - + // This only runs once for the site key loop if (!fftiSources.contains(fftiSource)) { FFTIProcessor ffti = new FFTIProcessor( @@ -735,16 +741,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } } - + SourceXML source = getSourceConfig().getSource( ffmpRec.getSourceName()); if (!source.getSourceType().equals( SOURCE_TYPE.GUIDANCE.getSourceType())) { - String sourceSiteDataKey = getSourceSiteDataKey(source, - dataKey, ffmpRec); + String sourceSiteDataKey = getSourceSiteDataKey( + source, dataKey, ffmpRec); ffmpData.remove(sourceSiteDataKey); - statusHandler.info("Removing from memory: "+sourceSiteDataKey); + statusHandler.info("Removing from memory: " + + sourceSiteDataKey); } } } @@ -970,8 +977,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements getAbsoluteSourceFileName(sourceId)); try { - sbl = SerializationUtil - .transformFromThrift(SourceBinList.class, FileUtil.file2bytes(f.getFile(), true)); + sbl = SerializationUtil.transformFromThrift(SourceBinList.class, + FileUtil.file2bytes(f.getFile(), true)); } catch (FileNotFoundException fnfe) { statusHandler.handle(Priority.ERROR, "Unable to locate file " + f.getName()); @@ -979,8 +986,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements statusHandler.handle(Priority.ERROR, "Unable to read file " + f.getName()); } catch (IOException ioe) { - statusHandler.handle(Priority.ERROR, "General IO problem with file " - + f.getName(), ioe); + statusHandler.handle(Priority.ERROR, + "General IO problem with file " + f.getName(), ioe); } return sbl; @@ -992,8 +999,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements * @return */ public String getAbsoluteSourceFileName(String sourceId) { - return productType + File.separator + "sources" + File.separator + sourceId - + ".bin"; + return productType + File.separator + "sources" + File.separator + + sourceId + ".bin"; } /** @@ -1098,7 +1105,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements */ public FFMPDataContainer getFFMPDataContainer(String siteSourceKey, - ArrayList hucs, Date backDate) { + Date backDate) { FFMPDataContainer container = ffmpData.get(siteSourceKey); @@ -1112,7 +1119,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements siteKey = parts[0]; } - container = loadFFMPDataContainer(siteSourceKey, hucs, siteKey, + container = loadFFMPDataContainer(siteSourceKey, siteKey, config.getCWA(), backDate); if (container != null) { @@ -1194,180 +1201,157 @@ public class FFMPGenerator extends CompositeProductGenerator implements /** * Process the ffmp data container + * * @param ffmpRec * @param productKey */ - public void processDataContainer(FFMPRecord ffmpRec, String productKey) { + public void processDataContainer(FFMPRecord ffmpRec, String productKey) { - String sourceName = null; - Date backDate = null; - String sourceSiteDataKey = null; - FFMPDataContainer fdc = null; - boolean write = true; + String sourceName = null; + Date backDate = null; + String sourceSiteDataKey = null; + FFMPDataContainer fdc = null; + boolean write = true; - try { - // write out the fast loader cache file - long ptime = System.currentTimeMillis(); - SourceXML source = getSourceConfig().getSource( - ffmpRec.getSourceName()); - String dataKey = ffmpRec.getDataKey(); + try { + // write out the fast loader cache file + long ptime = System.currentTimeMillis(); + SourceXML source = getSourceConfig().getSource( + ffmpRec.getSourceName()); + String dataKey = ffmpRec.getDataKey(); - if (source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { - sourceName = source.getDisplayName(); - sourceSiteDataKey = sourceName; - // FFG is so infrequent go back a day - backDate = new Date(config.getDate().getTime() - - (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME)); - } else { - sourceName = ffmpRec.getSourceName(); - sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() - + "-" + dataKey; - backDate = new Date(ffmpRec.getDataTime().getRefTime() - .getTime() - - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME)); - } + if (source.getSourceType().equals( + SOURCE_TYPE.GUIDANCE.getSourceType())) { + sourceName = source.getDisplayName(); + sourceSiteDataKey = sourceName; + // FFG is so infrequent go back a day + backDate = new Date(config.getDate().getTime() + - (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME)); + } else { + sourceName = ffmpRec.getSourceName(); + sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() + + "-" + dataKey; + backDate = new Date(ffmpRec.getDataTime().getRefTime() + .getTime() + - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME)); + } - // deal with setting of needed HUCS - ArrayList hucs = template.getTemplateMgr().getHucLevels(); + // pull from disk if there + fdc = getFFMPDataContainer(sourceSiteDataKey, backDate); - if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType()) - || source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { - hucs.clear(); - hucs.add(FFMPRecord.ALL); - } else { - hucs.remove(FFMPRecord.VIRTUAL); - } + // brand new or initial load up + if (fdc == null || !loadedData.contains(sourceSiteDataKey)) { - // pull from disk if there - fdc = getFFMPDataContainer(sourceSiteDataKey, hucs, backDate); + long time = System.currentTimeMillis(); + fdc = new FFMPDataContainer(sourceSiteDataKey); + fdc = FFTIProcessor.populateDataContainer(fdc, template, + backDate, ffmpRec.getDataTime().getRefTime(), + ffmpRec.getWfo(), source, ffmpRec.getSiteKey()); - // brand new or initial load up - if (fdc == null || !loadedData.contains(sourceSiteDataKey)) { + long time2 = System.currentTimeMillis(); + statusHandler.handle(Priority.DEBUG, + "Populated new source: in " + (time2 - time) + + " ms: source: " + sourceSiteDataKey); - long time = System.currentTimeMillis(); - fdc = new FFMPDataContainer(sourceSiteDataKey, hucs); - fdc = FFTIProcessor.populateDataContainer(fdc, template, hucs, - backDate, ffmpRec.getDataTime().getRefTime(), - ffmpRec.getWfo(), source, ffmpRec.getSiteKey()); + } else { - if (source.getSourceType().equals( - SOURCE_TYPE.GAGE.getSourceType()) - || source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { - hucs.clear(); - hucs.add(FFMPRecord.ALL); - } else { - hucs.remove(FFMPRecord.VIRTUAL); - } + long time = System.currentTimeMillis(); + // guidance sources are treated as a mosaic and are handled + // differently. They are force read at startup. + // This is the main line sequence a source will take when + // updated. + if (!source.getSourceType().equals( + SOURCE_TYPE.GUIDANCE.getSourceType())) { - long time2 = System.currentTimeMillis(); - statusHandler.handle(Priority.DEBUG, - "Populated new source: in " + (time2 - time) - + " ms: source: " + sourceSiteDataKey); + Date newDate = fdc.getNewest(); + Date oldDate = fdc.getOldest(); - } else { + if (newDate != null && oldDate != null) { + if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate + .getTime()) >= (source + .getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) { + // force a re-query back to the newest time in + // existing source container, this will fill in + // gaps + // if + // they exist. + fdc = FFTIProcessor.populateDataContainer(fdc, + template, newDate, ffmpRec.getDataTime() + .getRefTime(), ffmpRec.getWfo(), + source, ffmpRec.getSiteKey()); - long time = System.currentTimeMillis(); - // guidance sources are treated as a mosaic and are handled - // differently. They are force read at startup. - // This is the main line sequence a source will take when - // updated. - if (!source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { + } else if (oldDate + .after(new Date( + backDate.getTime() + - (source + .getExpirationMinutes(ffmpRec + .getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) { + // force a re-query back to barrierTime for + // existing source container, this happens if + // the + // ingest was turned off for some period of + // time. + fdc = FFTIProcessor.populateDataContainer(fdc, + template, backDate, oldDate, + ffmpRec.getWfo(), source, + ffmpRec.getSiteKey()); + } + } - Date newDate = fdc.getNewest(); - Date oldDate = fdc.getOldest(); + long time2 = System.currentTimeMillis(); + statusHandler.handle(Priority.DEBUG, + "Checked Source files: in " + (time2 - time) + + " ms: source: " + sourceSiteDataKey); + } + } - if (newDate != null && oldDate != null) { - if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate - .getTime()) >= (source - .getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) { - // force a re-query back to the newest time in - // existing source container, this will fill in - // gaps - // if - // they exist. - fdc = FFTIProcessor.populateDataContainer(fdc, - template, null, newDate, ffmpRec - .getDataTime().getRefTime(), - ffmpRec.getWfo(), source, ffmpRec - .getSiteKey()); + // add current record data - } else if (oldDate - .after(new Date( - backDate.getTime() - - (source - .getExpirationMinutes(ffmpRec - .getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) { - // force a re-query back to barrierTime for - // existing source container, this happens if - // the - // ingest was turned off for some period of - // time. - fdc = FFTIProcessor.populateDataContainer(fdc, - template, null, backDate, oldDate, - ffmpRec.getWfo(), source, - ffmpRec.getSiteKey()); - } - } + fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source, + ffmpRec.getBasinData(), ffmpRec.getSiteKey()); - long time2 = System.currentTimeMillis(); - statusHandler.handle(Priority.DEBUG, - "Checked Source files: in " + (time2 - time) - + " ms: source: " + sourceSiteDataKey); - } - } + // cache it temporarily for FFTI use + if (source.getSourceType().equals( + SOURCE_TYPE.GUIDANCE.getSourceType())) { + // only write last one + write = false; - // add current record data - for (String huc : hucs) { - fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source, - ffmpRec.getBasinData(huc), huc, ffmpRec.getSiteKey()); - } + if (!ffmpData.containsKey(sourceSiteDataKey)) { + ffmpData.put(sourceSiteDataKey, fdc); + } else { + ffmpData.replace(sourceSiteDataKey, fdc); + } + } - // cache it temporarily for FFTI use - if (source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { - // only write last one - write = false; + statusHandler.handle( + Priority.INFO, + "Processed FFMPDataContainer: in " + + (System.currentTimeMillis() - ptime) + + " ms: source: " + sourceSiteDataKey); + } catch (Exception e) { + statusHandler.handle(Priority.ERROR, + "Failed Processing FFMPDataContainer" + e.getMessage()); - if (!ffmpData.containsKey(sourceSiteDataKey)) { - ffmpData.put(sourceSiteDataKey, fdc); - } else { - ffmpData.replace(sourceSiteDataKey, fdc); - } - } + } finally { + // purge it up + if (fdc != null) { + // this is defensive for if errors get thrown + if (backDate == null) { + backDate = new Date((System.currentTimeMillis()) + - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME)); + } - statusHandler.handle( - Priority.INFO, - "Processed FFMPDataContainer: in " - + (System.currentTimeMillis() - ptime) - + " ms: source: " + sourceSiteDataKey); - } catch (Exception e) { - statusHandler.handle(Priority.ERROR, - "Failed Processing FFMPDataContainer" + e.getMessage()); + if (!fdc.isPurged()) { + fdc.purge(backDate); + } - } finally { - // purge it up - if (fdc != null) { - // this is defensive for if errors get thrown - if (backDate == null) { - backDate = new Date((System.currentTimeMillis()) - - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME)); - } - - if (!fdc.isPurged()) { - fdc.purge(backDate); - } - - if (write) { - // write it out - writeAggregateRecord(fdc, sourceSiteDataKey); - } - } - } - } + if (write) { + // write it out + writeAggregateRecord(fdc, sourceSiteDataKey); + } + } + } + } /** * load existing container @@ -1379,7 +1363,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements * @return */ public FFMPDataContainer loadFFMPDataContainer(String sourceSiteDataKey, - ArrayList hucs, String siteKey, String wfo, Date backDate) { + String siteKey, String wfo, Date backDate) { FFMPDataContainer fdc = null; FFMPAggregateRecord record = null; @@ -1396,7 +1380,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements // condition for first time read in if (fdc == null && record != null) { // creates a place holder for this source - fdc = new FFMPDataContainer(sourceSiteDataKey, hucs, record); + fdc = new FFMPDataContainer(sourceSiteDataKey, record); populated = true; } @@ -1404,9 +1388,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements if (record != null && !populated) { fdc.setAggregateData(record); } - - // sometimes a record will sit around for a long time and it will have data going back to the last precip event - // this can be an enormous amount of time. Want to get the data dumped from memory ASAP. + + // sometimes a record will sit around for a long time and it will have + // data going back to the last precip event + // this can be an enormous amount of time. Want to get the data dumped + // from memory ASAP. if (fdc != null) { fdc.purge(backDate); } @@ -1443,9 +1429,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements * * @param fdc */ - public void writeAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) { + public void writeAggregateRecord(FFMPDataContainer fdc, + String sourceSiteDataKey) { - WriteAggregateRecord writer = new WriteAggregateRecord(fdc, sourceSiteDataKey); + WriteAggregateRecord writer = new WriteAggregateRecord(fdc, + sourceSiteDataKey); writer.run(); } @@ -1458,7 +1446,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements private class WriteAggregateRecord implements Runnable { private FFMPDataContainer fdc; - + private String sourceSiteDataKey; public void run() { @@ -1470,10 +1458,12 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } - public WriteAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) { + public WriteAggregateRecord(FFMPDataContainer fdc, + String sourceSiteDataKey) { this.fdc = fdc; this.sourceSiteDataKey = sourceSiteDataKey; - statusHandler.handle(Priority.DEBUG, "Created Aggregate Record Writer"); + statusHandler.handle(Priority.DEBUG, + "Created Aggregate Record Writer"); } /** @@ -1492,34 +1482,37 @@ public class FFMPGenerator extends CompositeProductGenerator implements aggRecord.setWfo(config.getCWA()); // times for Guidance basins will be null aggRecord.setTimes(fdc.getOrderedTimes()); - - for (FFMPBasinData fbd : fdc.getBasinMap().values()) { - fbd.serialize(); - aggRecord.addBasinData(fbd); - } + fdc.getBasinData().serialize(); + aggRecord.setBasins(fdc.getBasinData()); } - if (aggRecord.getBasinsMap().size() > 0) { + if (aggRecord.getBasins() != null) { try { StorageProperties sp = null; String compression = PluginRegistry.getInstance() - .getRegisteredObject(productType).getCompression(); + .getRegisteredObject(productType) + .getCompression(); if (compression != null) { sp = new StorageProperties(); sp.setCompression(Compression.valueOf(compression)); } - - byte[] bytes = SerializationUtil.transformToThrift(aggRecord); - + + byte[] bytes = SerializationUtil + .transformToThrift(aggRecord); + // NAME | GROUP | array |Dimension | size - IDataRecord rec = new ByteDataRecord(sourceSiteDataKey, config.getCWA(), - bytes, 1, new long[] { bytes.length }); - - File hdf5File = FFMPUtils.getHdf5File(config.getCWA(), sourceSiteDataKey); - IDataStore dataStore = DataStoreFactory.getDataStore(hdf5File); - // write it, allowing, and in fact encouraging replacing the last one + IDataRecord rec = new ByteDataRecord(sourceSiteDataKey, + config.getCWA(), bytes, 1, + new long[] { bytes.length }); + + File hdf5File = FFMPUtils.getHdf5File(config.getCWA(), + sourceSiteDataKey); + IDataStore dataStore = DataStoreFactory + .getDataStore(hdf5File); + // write it, allowing, and in fact encouraging replacing + // the last one dataStore.addDataRecord(rec, sp); dataStore.store(StoreOp.OVERWRITE); @@ -1538,7 +1531,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } - @Override public synchronized void configChanged(MonitorConfigEvent fce) { @@ -1703,8 +1695,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements * @return */ public String getAbsoluteFFTIFileName(String fftiName) { - return productType + File.separator + "ffti" + File.separator + fftiName - + ".bin"; + return productType + File.separator + "ffti" + File.separator + + fftiName + ".bin"; } /** @@ -1734,7 +1726,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements public ConcurrentHashMap getFFTIDataContainer() { return fftiData; } - + /** * Get value for an individual piece of the puzzle * @@ -1748,8 +1740,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements public FFTIAccum getAccumulationForSite(String fftiSourceKey, String fftiSiteKey, String fftiDataKey, double duration, String unit) { - SourceXML ffmpSource = getSourceConfig() - .getSourceByDisplayName(fftiSourceKey); + SourceXML ffmpSource = getSourceConfig().getSourceByDisplayName( + fftiSourceKey); FFTIAccum accumulator = null; String siteDataKey = ffmpSource.getDisplayName() + "-" + fftiSiteKey + "-" + fftiDataKey; @@ -1775,24 +1767,23 @@ public class FFMPGenerator extends CompositeProductGenerator implements long cur = config.getDate().getTime(); long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR); Date backDate = new Date(cur - timeBack); - long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * TimeUtil.MILLIS_PER_MINUTE; + long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) + * TimeUtil.MILLIS_PER_MINUTE; FFMPDataContainer fdc = null; - ArrayList hucs = new ArrayList(); - hucs.add(FFMPRecord.ALL); - - fdc = getFFMPDataContainer(siteDataKey, hucs, backDate); + fdc = getFFMPDataContainer(siteDataKey, backDate); if (fdc != null) { - FFMPBasinData fbd = fdc.getBasinData(FFMPRecord.ALL); + FFMPBasinData fbd = fdc.getBasinData(); // go over the list of CWAs gathering the pfaf list ArrayList pfafs = new ArrayList(); ArrayList cwaList = config.fdm.getCwaList(); - Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(), duration, fftiSiteKey); + Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(), + duration, fftiSiteKey); if (!Double.isNaN(gap)) { for (Long key : fbd.getBasins().keySet()) { @@ -1831,14 +1822,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements } ffmpData.remove(siteDataKey); - statusHandler.info("Removing from memory: "+siteDataKey); + statusHandler.info("Removing from memory: " + siteDataKey); accumulator.setReset(false); writeFFTIData(siteDataKey, accumulator); } return accumulator; } - + /** * Gets the ratio and difference values for this site * @@ -1880,24 +1871,24 @@ public class FFMPGenerator extends CompositeProductGenerator implements long cur = config.getDate().getTime(); long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR); Date backDate = new Date(cur - timeBack); - long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE; + long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) + * TimeUtil.MILLIS_PER_MINUTE; // make sure we have data - Date ffgBackDate = new Date(config.getDate().getTime() - - (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME)); + Date ffgBackDate = new Date( + config.getDate().getTime() + - (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME)); String primarySource = fscm.getPrimarySource(ffmpQSource); ProductXML product = fscm.getProduct(primarySource); - ArrayList hucs = new ArrayList(); - hucs.add(FFMPRecord.ALL); - FFMPDataContainer guidContainer = getFFMPDataContainer( - ffgType, hucs, ffgBackDate); + FFMPDataContainer guidContainer = getFFMPDataContainer(ffgType, + ffgBackDate); long guidSourceExpiration = 0l; if (guidContainer == null) { - guidContainer = new FFMPDataContainer(ffgType, hucs); + guidContainer = new FFMPDataContainer(ffgType); } for (SourceXML iguidSource : product @@ -1905,7 +1896,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements if (guidSourceExpiration == 0l) { guidSourceExpiration = iguidSource - .getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE; + .getExpirationMinutes(qSiteKey) + * TimeUtil.MILLIS_PER_MINUTE; break; } } @@ -1918,15 +1910,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements + " " + qSourceKey + " " + " comparison."); return values; } - - String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-" + qSiteKey + "-"+ qSiteKey; - FFMPDataContainer qpeContainer = getFFMPDataContainer(qpeSiteSourceDataKey, hucs, backDate); + + String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-" + + qSiteKey + "-" + qSiteKey; + FFMPDataContainer qpeContainer = getFFMPDataContainer( + qpeSiteSourceDataKey, backDate); if (qpeContainer != null) { // go over the list of CWAs gathering the pfaf list ArrayList pfafs = new ArrayList(); ArrayList cwaList = config.fdm.getCwaList(); - FFMPBasinData fbd = qpeContainer.getBasinData(FFMPRecord.ALL); + FFMPBasinData fbd = qpeContainer.getBasinData(); for (Long key : fbd.getBasins().keySet()) { for (String cwa : cwaList) { @@ -1953,22 +1947,22 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } - Double gap = FFTI.getGap(qpeContainer, ffmpQSource, config.getDate(), duration, - qSiteKey); + Double gap = FFTI.getGap(qpeContainer, ffmpQSource, + config.getDate(), duration, qSiteKey); if (!Double.isNaN(gap)) { - List qpes = qpeContainer.getBasinData(FFMPRecord.ALL) + List qpes = qpeContainer.getBasinData() .getAccumValues(pfafs, backDate, config.getDate(), expirationTime, false); FFMPGuidanceInterpolation interpolator = new FFMPGuidanceInterpolation( - fscm, product, frcm.getRunner( - config.getCWA()).getProduct(qSiteKey), - primarySource, ffgType, qSiteKey); + fscm, product, frcm.getRunner(config.getCWA()) + .getProduct(qSiteKey), primarySource, + ffgType, qSiteKey); interpolator.setInterpolationSources(duration); - List guids = guidContainer.getBasinData(FFMPRecord.ALL) + List guids = guidContainer.getBasinData() .getGuidanceValues(pfafs, interpolator, guidSourceExpiration); @@ -1982,44 +1976,41 @@ public class FFMPGenerator extends CompositeProductGenerator implements // replace or insert it ffmpData.remove(qpeSiteSourceDataKey); - statusHandler.info("Removing from memory: "+qpeSiteSourceDataKey); + statusHandler.info("Removing from memory: " + qpeSiteSourceDataKey); values.setReset(false); writeFFTIData(siteDataKey, values); } return values; } - - - + /** - * Persist the record that has finished processing. - * This is different than other DAT tools. - * Other tools wait until all are finished processing - * before persisting. FFMP persists as it goes in order - * to lessen the data surge being sent to pypies. + * Persist the record that has finished processing. This is different than + * other DAT tools. Other tools wait until all are finished processing + * before persisting. FFMP persists as it goes in order to lessen the data + * surge being sent to pypies. * * @param record * @return */ private synchronized void persistRecord(FFMPRecord record) { - + // persist out this record try { - setPluginDataObjects(new FFMPRecord[]{record}); - setPluginDao(new FFMPDao(getCompositeProductType(), - template, fscm, config.getCWA())); + setPluginDataObjects(new FFMPRecord[] { record }); + setPluginDao(new FFMPDao(getCompositeProductType(), template, fscm, + config.getCWA())); persistRecords(); fireTopicUpdate(); // clear out pdos that are written pdos = null; } catch (PluginException e) { - statusHandler.handle(Priority.PROBLEM, "Couldn't persist the record.", e); + statusHandler.handle(Priority.PROBLEM, + "Couldn't persist the record.", e); } - + } - - + /** * Find siteSourceDataKey * @@ -2028,25 +2019,25 @@ public class FFMPGenerator extends CompositeProductGenerator implements * @param ffmpRec * @return */ - private String getSourceSiteDataKey(SourceXML source, String dataKey, FFMPRecord ffmpRec) { - + private String getSourceSiteDataKey(SourceXML source, String dataKey, + FFMPRecord ffmpRec) { + String sourceName = source.getSourceName(); String sourceSiteDataKey = null; - - if (source.getSourceType().equals( - SOURCE_TYPE.GUIDANCE.getSourceType())) { + + if (source.getSourceType().equals(SOURCE_TYPE.GUIDANCE.getSourceType())) { sourceName = source.getDisplayName(); sourceSiteDataKey = sourceName; - + } else { sourceName = ffmpRec.getSourceName(); - sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() - + "-" + dataKey; + sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() + "-" + + dataKey; } - + return sourceSiteDataKey; } - + /** * Log process statistics * @@ -2083,4 +2074,4 @@ public class FFMPGenerator extends CompositeProductGenerator implements } } - } +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java index f295d028ab..ed7ae1a1ae 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java @@ -20,7 +20,6 @@ package com.raytheon.uf.edex.plugin.ffmp.common; * further licensing information. **/ -import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map.Entry; @@ -45,6 +44,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator; * 29 July, 2012 578 dhladky memory work * 27 Jan, 2013 1478 dhladky Changed arraylist to list for times, more constants * 02/01/13 1569 D. Hladky Added constants + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @author dhladky @@ -93,15 +93,14 @@ public class FFMPInterpolatedGuidanceDelay { public boolean calculateDelayedGuidance() { boolean delayGuidance = false; - ArrayList hucs = new ArrayList(); - hucs.add(FFMPRecord.ALL); - FFMPDataContainer qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName() - + "-" + siteKey + "-" + siteKey, hucs, backDate); + FFMPDataContainer qpeContainer = generator.getFFMPDataContainer( + qpeSource.getSourceName() + "-" + siteKey + "-" + siteKey, + backDate); // Don't do anything, we have no QPE if (qpeContainer != null) { - + long expirationTime = qpeSource.getExpirationMinutes(siteKey) * TimeUtil.MILLIS_PER_MINUTE; // determine lag_time @@ -115,8 +114,8 @@ public class FFMPInterpolatedGuidanceDelay { .getOrderedTimes(currentRecord.getDataTime().getRefTime()); // EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld) - for (Entry entry : currentRecord.getBasinsMap() - .get(FFMPRecord.ALL).getBasins().entrySet()) { + for (Entry entry : currentRecord.getBasinData() + .getBasins().entrySet()) { FFMPBasin currBasin = entry.getValue(); FFMPGuidanceBasin oldBasin = (FFMPGuidanceBasin) previousGuidanceData .get(entry.getKey()); @@ -136,8 +135,8 @@ public class FFMPInterpolatedGuidanceDelay { // this is essentially a ratio of the first accumulation // step increment // to the total amount over this time window. - FFMPBasin qpeBasin = qpeContainer.getBasinData( - FFMPRecord.ALL).get(entry.getKey()); + FFMPBasin qpeBasin = qpeContainer.getBasinData().get( + entry.getKey()); if (qpeBasin != null) { float intervalAccum = qpeBasin.getAccumValue( @@ -160,7 +159,7 @@ public class FFMPInterpolatedGuidanceDelay { currBasin.setValue(newDate, val); } } - + delayGuidance = true; } diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java index 17da3e3e7f..5d467b9b80 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java @@ -98,6 +98,7 @@ import com.vividsolutions.jts.geom.Polygon; * 02/01/13 1569 D. Hladky Added constants * 02/25/13 1660 D. Hladky FFTI design change to help mosaic processing. * 05/01/2013 15684 zhao Unlock when Exception caught + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * * @author dhladky @@ -214,29 +215,6 @@ public class FFMPProcessor { processSource(); } - // don't do gages for anything other than "ALL" - if ((sourceId != null) - && !source.getSourceType().equals( - FFMPSourceConfigurationManager.SOURCE_TYPE.GAGE - .getSourceType())) { - - //ArrayList hucs = template.getTemplateMgr().getHucLevels();// DR 15514 - String[] hucs = template.getTemplateMgr().getHucLevelsInArray();// DR 15514 - synchronized (hucs) { - if (hucs != null) { - //for (String huc : hucs) { - for(int i=0; i hucs = new ArrayList(); - hucs.add("ALL"); + FFMPDataContainer ffgContainer = generator - .getFFMPDataContainer(sourceNameString, hucs, + .getFFMPDataContainer(sourceNameString, backDate); if (ffgContainer != null @@ -688,7 +664,7 @@ public class FFMPProcessor { siteKey, guidFrequency, source, qpeSource, previousDate, recdate, generator, - ffgContainer.getBasinData(FFMPRecord.ALL), + ffgContainer.getBasinData(), ffmpRec); boolean delayGuidance = figd @@ -811,9 +787,9 @@ public class FFMPProcessor { * * @param ffmp */ - private FFMPBasinData getBasinData(String huc) { + private FFMPBasinData getBasinData() { - return ffmpRec.getBasinData(huc); + return ffmpRec.getBasinData(); } /** @@ -823,15 +799,11 @@ public class FFMPProcessor { * @param huc * @return */ - private FFMPBasin getBasin(Long pfaf, String huc) { - FFMPBasin basin = getBasinData(huc).get(pfaf); + private FFMPBasin getBasin(Long pfaf) { + FFMPBasin basin = getBasinData().get(pfaf); if (basin == null) { - if (huc.equals(FFMPRecord.ALL)) { - basin = new FFMPBasin(pfaf, false); - } else { - basin = new FFMPBasin(pfaf, true); - } - getBasinData(huc).put(pfaf, basin); + basin = new FFMPBasin(pfaf, false); + getBasinData().put(pfaf, basin); } return basin; } @@ -845,64 +817,15 @@ public class FFMPProcessor { */ private FFMPVirtualGageBasin getVirtualBasin(String lid, Long pfaf, String huc) { - FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData(huc) + FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData() .get(pfaf); if (basin == null) { - if (huc.equals(FFMPRecord.ALL)) { - basin = new FFMPVirtualGageBasin(lid, pfaf, false); - } else { - basin = new FFMPVirtualGageBasin(lid, pfaf, true); - } - getBasinData(huc).put(pfaf, basin); + basin = new FFMPVirtualGageBasin(lid, pfaf, false); + getBasinData().put(pfaf, basin); } return basin; } - /** - * Sets the values for the aggregated basins - * - * @param type - * @return - */ - private void setValues(String huc) { - - try { - // Get basins for level, we process VGB's differently because it is - // a - // special case - if (!huc.equals(FFMPRecord.VIRTUAL) && !huc.equals(FFMPRecord.ALL)) { - - for (DomainXML domain : template.getDomains()) { - - LinkedHashMap map = template.getMap(siteKey, - domain.getCwa(), huc); - - for (Long pfaf : map.keySet()) { - FFMPBasin basin = getBasin(pfaf, huc); - Float val = 0.0f; - // average values - try { - ArrayList aggPfafs = template - .getAggregatePfafs(pfaf, siteKey, huc); - ArrayList areas = template - .getAreas(aggPfafs); - val = ffmpRec.getBasinData(FFMPRecord.ALL).getAverageValue( - aggPfafs, areas); - } catch (Exception e) { - // Value is NAN, ignore it. - } - - basin.setValue(config.getDate(), val); - } - } - } - - } catch (Exception e) { - config.getGenerator().logger.error("Unable to process " + huc - + " level data"); - } - } - /** * Process the PDO for a geometry * diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java index adf00a87fd..c72fc35258 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java @@ -57,6 +57,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator; * * Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp * Apr 18, 2013 1919 dhladky Fixed VGB breakage + * Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL * * @author dhladky * @version 1.0 @@ -182,8 +183,8 @@ public class FFTIProcessor { */ public static FFMPDataContainer populateDataContainer( FFMPDataContainer sourceContainer, FFMPTemplates template, - ArrayList hucs, Date startDate, Date endDate, String wfo, - SourceXML source, String siteKey) { + Date startDate, Date endDate, String wfo, SourceXML source, + String siteKey) { ArrayList uris = getUris(startDate, endDate, wfo, source, siteKey); @@ -209,19 +210,11 @@ public class FFTIProcessor { if (!contains) { try { - if (hucs == null) { - hucs = new ArrayList(); - hucs.add(FFMPRecord.ALL); - } - - for (String huc : hucs) { - - rec = populateRecord(rec, huc, template); - FFMPBasinData newData = rec.getBasinData(huc); - sourceContainer.addFFMPEntry(rec.getDataTime() - .getRefTime(), source, newData, huc, siteKey); - - } + rec = populateRecord(rec, template); + FFMPBasinData newData = rec.getBasinData(); + sourceContainer.addFFMPEntry( + rec.getDataTime().getRefTime(), source, newData, + siteKey); // System.out.println("Adding Time: " // + rec.getDataTime().getRefTime()); @@ -309,8 +302,7 @@ public class FFTIProcessor { * @return * @throws PluginException */ - public static FFMPRecord populateRecord(FFMPRecord rec, String huc, - FFMPTemplates template) throws PluginException { + public static FFMPRecord populateRecord(FFMPRecord rec, FFMPTemplates template) throws PluginException { try { @@ -328,9 +320,9 @@ public class FFTIProcessor { // check for gage(VGB) types, if so process as a VGB if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType())) { - rec.retrieveVirtualMapFromDataStore(template, huc); + rec.retrieveVirtualMapFromDataStore(template); } else { - rec.retrieveMapFromDataStore(template, huc); + rec.retrieveMapFromDataStore(template); } // System.out.println("Size of huc: "