xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2184 Removed all other HUCs other than ALL from storage" into omaha_13.5.1

Browse source 
Former-commit-id: 96ae856660 [formerly 09cae0cba3] [formerly 3649af1b71 [formerly 7f9e83b42df42c1bec7946833e88a8916fc0c12d]]
Former-commit-id: 3649af1b71
Former-commit-id: cb9b26a380
This commit is contained in:
Nate Jensen 2013-07-17 16:34:25 -05:00 • committed by Gerrit Code Review
commit a9f0243e01
20 changed files with 1087 additions and 1178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java
View file

@ -97,6 +97,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen No longer starts loading threads, resourceData does that
* Jun 07, 2013 2075 njensen Extracted FFMPProcessUris to separate class
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -291,11 +292,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @param phuc
* @return
*/
protected List<String> getLoadedUris(String siteKey, String source,
String phuc) {
protected List<String> getLoadedUris(String siteKey, String source) {
FFMPSiteData siteData = siteDataMap.get(siteKey);
FFMPSourceData sourceData = siteData.getSourceData(source);
return sourceData.getLoadedUris(phuc);
return sourceData.getLoadedUris();
}
/**
@ -308,7 +308,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @throws VizException
*/
public void populateFFMPRecord(String siteKey, String dataKey,
String source, Date ptime, String phuc, boolean retrieveNew) {
String source, Date ptime, boolean retrieveNew) {
if (source != null) {
boolean dupOverride = false;
@ -325,11 +325,11 @@ public class FFMPMonitor extends ResourceMonitor {
for (String uri : uris) {
if (uri != null) {
if (dupOverride
|| !getLoadedUris(siteKey, source, phuc)
|| !getLoadedUris(siteKey, source)
.contains(uri)) {
try {
populateFFMPRecord(siteKey, new FFMPRecord(
uri), source, phuc);
uri), source);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, "
@ -353,10 +353,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @throws VizException
*/
public FFMPRecord populateFFMPRecord(String uri, String siteKey,
String source, String phuc) throws Exception {
String source) throws Exception {
try {
populateFFMPRecord(siteKey, new FFMPRecord(uri), source, phuc);
populateFFMPRecord(siteKey, new FFMPRecord(uri), source);
} catch (VizException e) {
statusHandler.handle(Priority.INFO,
"FFMP Can't retrieve FFMP URI, " + uri, e);
@ -387,31 +387,26 @@ public class FFMPMonitor extends ResourceMonitor {
FFMPRecord curRecord = sourceData.getRecord();
if (curRecord == null) {
// add each huc requested
for (String huc : data.getBasinsMap().keySet()) {
// add all of the uris
for (Entry<Date, List<String>> duris : uris.entrySet()) {
if (data.getTimes().contains(duris.getKey().getTime())) {
for (String uri : duris.getValue()) {
if (curRecord == null) {
curRecord = new FFMPRecord(uri);
sourceData.setRecord(curRecord);
}
sourceData.addLoadedUri(huc, uri);
// add all of the uris
for (Entry<Date, List<String>> duris : uris.entrySet()) {
if (data.getTimes().contains(duris.getKey().getTime())) {
for (String uri : duris.getValue()) {
if (curRecord == null) {
curRecord = new FFMPRecord(uri);
sourceData.setRecord(curRecord);
}
sourceData.addLoadedUri(uri);
}
}
}
}
if (curRecord != null) {
for (Entry<String, FFMPBasinData> entry : data.getBasinsMap()
.entrySet()) {
FFMPBasinData basinData = entry.getValue();
basinData.populate(data.getTimes());
curRecord.populate(basinData, entry.getKey());
}
FFMPBasinData basinData = data.getBasins();
basinData.populate(data.getTimes());
curRecord.populate(basinData);
}
}
@ -422,13 +417,13 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey
* @param ffmpRec
* @param source
* @param phuc
*
* @throws Exception
*/
public void populateFFMPRecord(String siteKey, FFMPRecord ffmpRec,
String source, String phuc) throws Exception {
String source) throws Exception {
FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source, phuc);
FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source);
flr.run();
}
@ -446,12 +441,12 @@ public class FFMPMonitor extends ResourceMonitor {
String source, String phuc, FFMPBasin basin) throws VizException {
if (dataUri != null) {
List<String> uris = getLoadedUris(siteKey, source, phuc);
List<String> uris = getLoadedUris(siteKey, source);
if (!uris.contains(dataUri)) {
try {
SourceXML sourceXML = fscm.getSource(source);
FFMPRecord ffmpRec = populateFFMPRecord(dataUri, siteKey,
source, phuc);
source);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory.getDataStore(loc);
@ -698,14 +693,12 @@ public class FFMPMonitor extends ResourceMonitor {
*
*/
public FFMPRecord getFFMPData(ProductXML product, String siteKey,
String dataKey, String sourceName, Date ptime, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date ptime, boolean retrieveNew) {
FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName)
.getRecord();
if ((record != null)
&& (record.getBasinData(phuc).getBasins().size() > 0)) {
if ((record != null) && (record.getBasinData().getBasins().size() > 0)) {
SourceXML sourceXML = getSourceConfig().getSource(sourceName);
@ -731,8 +724,7 @@ public class FFMPMonitor extends ResourceMonitor {
continue;
} else {
populateFFMPRecord(siteKey, dataKey,
source.getSourceName(), ptime, phuc,
retrieveNew);
source.getSourceName(), ptime, retrieveNew);
}
}
} else {
@ -740,7 +732,7 @@ public class FFMPMonitor extends ResourceMonitor {
if (!siteDataMap.get(siteKey).getSourceData(sourceName)
.hasLoadedAnyUris()) {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
}
@ -748,7 +740,7 @@ public class FFMPMonitor extends ResourceMonitor {
.getRecord();
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime, phuc,
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
retrieveNew);
}
@ -775,17 +767,17 @@ public class FFMPMonitor extends ResourceMonitor {
.getGuidanceSources(product, guidSrc)) {
populateFFMPRecord(siteKey, dataKey,
ffgSource.getSourceName(), ptime, phuc,
ffgSource.getSourceName(), ptime,
retrieveNew);
}
} else {
populateFFMPRecord(siteKey, dataKey, sourceName,
ptime, phuc, retrieveNew);
ptime, retrieveNew);
}
}
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
} else {
// special case where FFG is the primary source
@ -799,7 +791,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = sourcexml.getDisplayName();
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
}
@ -1054,20 +1046,18 @@ public class FFMPMonitor extends ResourceMonitor {
* Get the pertinent QPE source Record.
*
* @param date
* @param phuc
* @param retrieveNew
* @return
*/
public FFMPRecord getQPERecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date date, boolean retrieveNew) {
// comparisons done with table display
if (product != null) {
sourceName = product.getQpe();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1095,12 +1085,11 @@ public class FFMPMonitor extends ResourceMonitor {
* Get the rate record.
*
* @param date
* @param phuc
* @param retrieveNew
* @return
*/
public FFMPRecord getRateRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
String dataKey, String sourceName, Date date,
boolean retrieveNew) {
// comparisons done with table display
@ -1108,7 +1097,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = product.getRate();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1142,8 +1131,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public FFMPRecord getQPFRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date date, boolean retrieveNew) {
FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance()
.getTableConfigData(siteKey);
@ -1157,7 +1145,7 @@ public class FFMPMonitor extends ResourceMonitor {
.getSourceName();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1210,8 +1198,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = source.getDisplayName();
}
return getFFMPData(product, siteKey, null, sourceName, date, phuc,
false);
return getFFMPData(product, siteKey, null, sourceName, date, false);
}
/**
@ -1226,7 +1213,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public Map<String, FFMPRecord> getGuidanceRecords(ProductXML product,
String siteKey, Date date, String phuc, boolean retrieveNew) {
String siteKey, Date date, boolean retrieveNew) {
Map<String, FFMPRecord> guidRecs = new HashMap<String, FFMPRecord>();
ProductRunXML productRun = FFMPRunConfigurationManager.getInstance()
@ -1236,7 +1223,7 @@ public class FFMPMonitor extends ResourceMonitor {
for (String type : guidTypes) {
FFMPRecord guidRec = getFFMPData(product, siteKey, null, type,
date, phuc, retrieveNew);
date, retrieveNew);
guidRecs.put(type, guidRec);
}
@ -1275,7 +1262,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public FFMPRecord getVirtualRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
String dataKey, String sourceName, Date date,
boolean retrieveNew) {
// comparisons done with table display
// field doesn't matter here
@ -1284,7 +1271,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = product.getVirtual();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1305,7 +1292,7 @@ public class FFMPMonitor extends ResourceMonitor {
FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName)
.getRecord();
if (record != null) {
FFMPBasinData basinData = record.getBasinData(phuc);
FFMPBasinData basinData = record.getBasinData();
if (basinData != null) {
basin = basinData.get(pfaf);
}
@ -1350,7 +1337,7 @@ public class FFMPMonitor extends ResourceMonitor {
.get(siteKey)
.getSourceData(
source.getSourceName())
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey,
source.getSourceName(), phuc,
@ -1376,7 +1363,7 @@ public class FFMPMonitor extends ResourceMonitor {
for (String uri : uris) {
if (!siteDataMap.get(siteKey)
.getSourceData(sourceName)
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey, sourceName,
phuc, fgb);
@ -1397,7 +1384,7 @@ public class FFMPMonitor extends ResourceMonitor {
POINT_RETRIVAL: for (List<String> uris : availableUris.values()) {
for (String uri : uris) {
if (!siteDataMap.get(siteKey).getSourceData(sourceName)
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey, sourceName, phuc,
basin);
@ -1593,21 +1580,15 @@ public class FFMPMonitor extends ResourceMonitor {
.hasLoadedAnyUris()) {
FFMPSourceData sourceData = siteData
.getSourceData(sourceName);
Set<String> hucs = sourceData
.getLoadedHucs();
for (String huc : hucs) {
sourceData.getLoadedUris(huc).remove(
uri);
}
sourceData.removeLoadedUri(uri);
}
}
} else {
FFMPSourceData sourceData = siteData
.getSourceData(fsource);
Set<String> hucs = sourceData.getLoadedHucs();
for (String huc : hucs) {
sourceData.getLoadedUris(huc).remove(uri);
}
sourceData.removeLoadedUri(uri);
}
}
}
@ -1625,16 +1606,14 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey
* @param sourceName
* @param barrierTime
* @param phuc
*/
public void processUri(String uri, String siteKey, String sourceName,
Date barrierTime, String phuc) {
Date barrierTime) {
if (uri != null) {
try {
FFMPRecord record = populateFFMPRecord(uri, siteKey,
sourceName, phuc);
FFMPRecord record = populateFFMPRecord(uri, siteKey, sourceName);
if (record != null) {
record.getBasinData(phuc).loadNow();
record.getBasinData().loadNow();
SourceXML source = getSourceConfig().getSource(sourceName);
if (source != null) {
record.setExpiration(source
@ -1656,10 +1635,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @param sourceName
*/
public void processUris(NavigableMap<Date, List<String>> uriMap,
String siteKey, String sourceName, Date barrierTime, String phuc,
String siteKey, String sourceName, Date barrierTime,
SubMonitor smonitor) {
FFMPProcessUris processor = new FFMPProcessUris(this, uriMap, siteKey,
sourceName, barrierTime, phuc);
sourceName, barrierTime);
processor.run(smonitor);
}
@ -2126,16 +2105,13 @@ public class FFMPMonitor extends ResourceMonitor {
final String fsource;
final String fhuc;
final String fsiteKey;
public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec,
String source, String huc) throws Exception {
public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec, String source)
throws Exception {
this.fffmpRec = ffmpRec;
this.fsource = source;
this.fsiteKey = siteKey;
this.fhuc = huc;
}
public void run() {
@ -2143,7 +2119,7 @@ public class FFMPMonitor extends ResourceMonitor {
load();
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM, "FFMP load FFMPData, "
+ fsource + " " + fhuc, e);
+ fsource, e);
}
}
@ -2151,7 +2127,7 @@ public class FFMPMonitor extends ResourceMonitor {
if (fffmpRec != null) {
List<String> uris = getLoadedUris(fsiteKey, fsource, fhuc);
List<String> uris = getLoadedUris(fsiteKey, fsource);
String dataUri = fffmpRec.getDataURI();
if (!uris.contains(dataUri)) {
Date refTime = fffmpRec.getDataTime().getRefTime();
@ -2188,20 +2164,20 @@ public class FFMPMonitor extends ResourceMonitor {
try {
if (isGageSource && fhuc.equals(FFMPRecord.ALL)) {
if (isGageSource) {
curRecord.retrieveVirtualMapFromDataStore(loc,
dataUri, getTemplates(fsiteKey), refTime,
fffmpRec.getSourceName());
} else {
curRecord.retrieveMapFromDataStore(loc, dataUri,
getTemplates(fffmpRec.getSiteKey()), fhuc,
getTemplates(fffmpRec.getSiteKey()),
refTime, fffmpRec.getSourceName());
}
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, " + dataUri, e);
}
sourceData.addLoadedUri(fhuc, dataUri);
sourceData.addLoadedUri(dataUri);
}
}
}

12
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java
View file

@ -44,6 +44,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 7, 2013 njensen Initial creation
* Jul 15, 2013 2184 dhladky Removed all HUC's but ALL
*
* </pre>
*
@ -64,18 +65,15 @@ public class FFMPProcessUris {
private final Date fbarrierTime;
private final String fhuc;
private final FFMPMonitor ffmpMonitor;
public FFMPProcessUris(FFMPMonitor ffmpMonitor,
NavigableMap<Date, List<String>> uriMap, String siteKey,
String sourceName, Date barrierTime, String phuc) {
String sourceName, Date barrierTime) {
this.furiMap = uriMap;
this.fsiteKey = siteKey;
this.fbarrierTime = barrierTime;
this.fsourceName = sourceName;
this.fhuc = phuc;
this.ffmpMonitor = ffmpMonitor;
}
@ -90,7 +88,7 @@ public class FFMPProcessUris {
isGuidance = true;
}
List<String> loadedUris = ffmpMonitor.getLoadedUris(fsiteKey,
fsourceName, fhuc);
fsourceName);
Set<FFMPRecord> populatedRecords = new HashSet<FFMPRecord>();
for (List<String> uris : furiMap.descendingMap().values()) {
for (String uri : uris) {
@ -102,7 +100,7 @@ public class FFMPProcessUris {
|| isGuidance) {
try {
record = ffmpMonitor.populateFFMPRecord(uri,
fsiteKey, fsourceName, fhuc);
fsiteKey, fsourceName);
if (record != null) {
populatedRecords.add(record);
if (source != null) {
@ -121,7 +119,7 @@ public class FFMPProcessUris {
monitor.beginTask(null, populatedRecords.size());
for (FFMPRecord record : populatedRecords) {
record.getBasinData(fhuc).loadNow();
record.getBasinData().loadNow();
monitor.worked(1);
}
}

67
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java
View file

@ -20,11 +20,9 @@
package com.raytheon.uf.viz.monitor.ffmp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
@ -42,6 +40,7 @@ import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
* ------------ ---------- ----------- --------------------------
* Feb 18, 2013 njensen Initial creation
* Feb 28, 2013 1729 dhladky Sped up, synch blocks were hanging it.
* Jul 15, 2013 2184 dhladky Removed all HUC's but ALL
*
* </pre>
*
@ -59,7 +58,7 @@ public class FFMPSourceData {
private ConcurrentNavigableMap<Date, List<String>> availableUris = new ConcurrentSkipListMap<Date, List<String>>();
/** map of huc to list of loaded URIs **/
private ConcurrentMap<String, List<String>> loadedUris = new ConcurrentHashMap<String, List<String>>();
private List<String> loadedUris = new ArrayList<String>();
/**
* Clears the data
@ -68,7 +67,9 @@ public class FFMPSourceData {
ffmpData = null;
previousUriQueryDate = null;
availableUris.clear();
loadedUris.clear();
synchronized (loadedUris) {
loadedUris.clear();
}
}
/**
@ -110,45 +111,23 @@ public class FFMPSourceData {
}
/**
* Gets the URIs associated with a HUC that have been loaded.
* Gets the URIs that have been loaded.
*
* @param huc
* @return
*/
public List<String> getLoadedUris(String huc) {
List<String> loaded = loadedUris.get(huc);
if (loaded == null) {
loaded = new ArrayList<String>();
List<String> previous = loadedUris.putIfAbsent(huc, loaded);
if (previous != null) {
return previous;
}
}
return loaded;
public List<String> getLoadedUris() {
return Collections.unmodifiableList(loadedUris);
}
/**
* Tracks a URI associated with a HUC as loaded.
* Tracks a URI as loaded.
*
* @param huc
* @param uri
*/
public void addLoadedUri(String huc, String uri) {
List<String> uriList = loadedUris.get(huc);
if (uriList == null) {
uriList = new ArrayList<String>();
List<String> previous = loadedUris.putIfAbsent(huc, uriList);
if (previous != null) {
uriList = previous;
}
public void addLoadedUri(String uri) {
synchronized (loadedUris) {
loadedUris.add(uri);
}
uriList.add(uri);
}
/**
@ -160,15 +139,6 @@ public class FFMPSourceData {
return !loadedUris.isEmpty();
}
/**
* Gets the set of HUCs that have loaded some URIs.
*
* @return
*/
public Set<String> getLoadedHucs() {
return loadedUris.keySet();
}
/**
* Gets the Available URIs based on time.
*
@ -178,4 +148,15 @@ public class FFMPSourceData {
return availableUris;
}
/**
* Removes a URI
*
* @param uri
*/
public void removeLoadedUri(String uri) {
synchronized (loadedUris) {
loadedUris.remove(uri);
}
}
}

245
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java
View file

@ -77,6 +77,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData;
* May 22, 2013 1902 mpduff Code cleanup.
* Jun 11, 2013 2085 njensen Extracted row creation to FFMPRowGenerator and
* multi-threaded row creation.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -164,134 +165,139 @@ public class FFMPDataGenerator {
public FFMPTableData generateFFMPData() throws Exception {
// You should always have at least a QPE data source
FFMPTableData tData = null;
// update the FFFGDataManager
FFFGDataMgr.getUpdatedInstance();
try {
FIELDS field = getBaseField();
if (field == null || baseRec == null) {
return tData;
}
FFMPBasinData fbd = null;
if (centeredAggregationKey != null) {
fbd = baseRec.getBasinData(FFMPRecord.ALL);
} else {
fbd = baseRec.getBasinData(huc);
}
tData = new FFMPTableData(fbd.getBasins().size());
List<DomainXML> domains = resource.getDomains();
if (!fbd.getBasins().isEmpty()) {
if ((centeredAggregationKey == null)
|| huc.equals(FFMPRecord.ALL)) {
// System.out.println(fbd.getBasins().keySet().size()
// + " rows in the table");
if ((centeredAggregationKey == null) || huc.equals(FFMPRecord.ALL)) {
// System.out.println(fbd.getBasins().keySet().size()
// + " rows in the table");
if (huc.equals(FFMPRecord.ALL)) {
FFMPBasinData fbd = baseRec.getBasinData();
tData = new FFMPTableData(fbd.getBasins().size());
for (Long key : fbd.getBasins().keySet()) {
if (huc.equals(FFMPRecord.ALL)) {
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb == null) {
continue;
}
for (DomainXML domain : domains) {
String cwa = domain.getCwa();
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb == null) {
continue;
}
if ((cwa.equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
try {
setFFMPRow(fbd.get(key), tData, false,
cwa);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(
virtualBasin.get(id),
tData, true, cwa);
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Couldn't create table row"
+ e);
}
}
}
for (DomainXML domain : domains) {
String cwa = domain.getCwa();
if ((cwa.equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
try {
setFFMPRow(fbd.get(key), tData, false, cwa);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
}
} else {
/*
* make sure at least one basin in the agg is in the
* CWA
*/
List<Long> pfafs = ft.getAggregatePfafs(key,
siteKey, huc);
boolean isVGB = false;
if (ft.checkVGBsInAggregate(key, siteKey, huc)) {
isVGB = true;
}
if (!pfafs.isEmpty()) {
FFMPBasinMetaData fmdb = ft.getBasinInDomains(
siteKey, domains, pfafs);
if (fmdb != null) {
try {
setFFMPRow(fbd.get(key), tData, isVGB,
null);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(virtualBasin.get(id),
tData, true, cwa);
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Couldn't create table row"
+ e);
}
}
}
}
}
}
}
// show pfafs in aggregation
else {
for (Long key : resource.getCenteredAggregatePfafs()) {
// Find all of the basins for this HUC level
List<Long> keyList = ft
.getHucKeyList(siteKey, huc, domains);
tData = new FFMPTableData(keyList.size());
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
for (Long key : keyList) {
if (fmdb != null) {
for (DomainXML domain : domains) {
List<Long> pfafs = ft.getAggregatePfafs(key, siteKey,
huc);
if ((domain.getCwa().equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
boolean isVGB = false;
if (ft.checkVGBsInAggregate(key, siteKey, huc)) {
isVGB = true;
}
setFFMPRow(fbd.get(key), tData, false, null);
if (!pfafs.isEmpty()) {
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(
virtualBasin.get(id),
tData, true, null);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row",
e);
}
FFMPBasinMetaData fmdb = ft.getBasinInDomains(
siteKey, domains, pfafs);
if (fmdb != null) {
try {
FFMPBasin basin = new FFMPBasin(key, true);
setFFMPRow(basin, tData, isVGB, null);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
}
}
}
}
}
// show pfafs in aggregation
else {
FFMPBasinData fbd = baseRec.getBasinData();
tData = new FFMPTableData(resource.getCenteredAggregatePfafs().size());
for (Long key : resource.getCenteredAggregatePfafs()) {
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb != null) {
for (DomainXML domain : domains) {
if ((domain.getCwa().equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
setFFMPRow(fbd.get(key), tData, false, null);
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, FFMPRecord.ALL,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(virtualBasin.get(id),
tData, true, null);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row",
e);
}
}
}
@ -333,7 +339,6 @@ public class FFMPDataGenerator {
Date tableTime = resource.getTableTime();
FIELDS field = null;
String localHuc = null;
FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance()
.getTableConfigData(siteKey);
@ -358,37 +363,27 @@ public class FFMPDataGenerator {
monitor.setQpeWindow(new FFMPTimeWindow(tableTime, qpeTime));
if (isWorstCase || (centeredAggregationKey != null)) {
// make sure that "ALL" is loaded
localHuc = FFMPRecord.ALL;
} else {
localHuc = huc;
}
FFMPRecord rateRecord = monitor.getRateRecord(product, siteKey,
dataKey, product.getRate(), paintRefTime, localHuc, true);
dataKey, product.getRate(), paintRefTime, true);
FFMPRecord qpeRecord = monitor.getQPERecord(product, siteKey, dataKey,
product.getQpe(), tableTime, localHuc, true);
product.getQpe(), tableTime, true);
FFMPRecord qpfRecord = monitor.getQPFRecord(product, siteKey, dataKey,
null, paintRefTime, localHuc, true);
null, paintRefTime, true);
guidRecords = monitor.getGuidanceRecords(product, siteKey, tableTime,
localHuc, true);
FFMPRecord virtualRecord = null;
if (localHuc.equals(FFMPRecord.ALL)) {
virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey,
product.getVirtual(), tableTime, localHuc, true);
}
true);
FFMPRecord virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey,
product.getVirtual(), tableTime, true);
try {
if (rateRecord != null) {
rateBasin = rateRecord.getBasinData(localHuc);
rateBasin = rateRecord.getBasinData();
if (!rateBasin.getBasins().isEmpty()) {
field = FIELDS.RATE;
baseRec = rateRecord;
}
}
if (qpeRecord != null) {
qpeBasin = qpeRecord.getBasinData(localHuc);
qpeBasin = qpeRecord.getBasinData();
if (!qpeBasin.getBasins().isEmpty()) {
field = FIELDS.QPE;
if (baseRec == null) {
@ -397,21 +392,21 @@ public class FFMPDataGenerator {
}
}
if (qpfRecord != null) {
qpfBasin = qpfRecord.getBasinData(localHuc);
qpfBasin = qpfRecord.getBasinData();
}
if (guidRecords != null) {
guidBasins = new HashMap<String, FFMPBasinData>();
for (String type : guidRecords.keySet()) {
if (guidRecords.get(type) != null) {
guidBasins.put(type, guidRecords.get(type)
.getBasinData(localHuc));
.getBasinData());
} else {
guidBasins.put(type, null);
}
}
}
if (virtualRecord != null) {
virtualBasin = virtualRecord.getBasinData(localHuc);
virtualBasin = virtualRecord.getBasinData();
}
// Get interpolators

187
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java
View file

@ -169,6 +169,8 @@ import com.vividsolutions.jts.geom.Point;
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen No longer schedules load threads,
* refactored updates
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
*
* </pre>
*
@ -598,15 +600,12 @@ public class FFMPResource extends
private FFMPBasin getBasin(Long key, FFMPRecord.FIELDS bfield,
Date recentTime, boolean aggregate) throws VizException {
FFMPBasin basin = null;
String huc = null;
if (aggregate) {
huc = getHuc();
basin = new FFMPBasin(key, aggregate);
} else {
huc = FFMPRecord.ALL;
basin = getRecord(bfield, recentTime).getBasinData().getBasins()
.get(key);
}
basin = getRecord(bfield, recentTime).getBasinData(huc).getBasins()
.get(key);
return basin;
}
@ -680,14 +679,14 @@ public class FFMPResource extends
break;
}
case RATE: {
value = getRateRecord(recentTime).getBasinData(
FFMPRecord.ALL).getMaxValue(pfafs, recentTime);
value = getRateRecord(recentTime).getBasinData()
.getMaxValue(pfafs, recentTime);
break;
}
case QPF: {
value = getQpfRecord(recentTime).getBasinData(
FFMPRecord.ALL).getAverageMaxValue(pfafs,
recentTime, getQpfSourceExpiration());
value = getQpfRecord(recentTime).getBasinData()
.getAverageMaxValue(pfafs, recentTime,
getQpfSourceExpiration());
break;
}
case GUIDANCE: {
@ -695,21 +694,17 @@ public class FFMPResource extends
.getCountyFipsByPfaf(pfafs.get(0));
value = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getMaxGuidanceValue(pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration(getFFGName()), fips);
break;
}
case QPE: {
value = getQpeRecord().getBasinData(FFMPRecord.ALL)
.getAccumMaxValue(
pfafs,
recentTime,
getTableTime(),
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
value = getQpeRecord().getBasinData().getAccumMaxValue(
pfafs, recentTime, getTableTime(),
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML().isRate());
break;
}
}
@ -737,32 +732,64 @@ public class FFMPResource extends
break;
}
case RATE:
value = getBasin(key, field, recentTime, aggregate)
.getValue(recentTime);
if (aggregate) {
value = getRateRecord(recentTime).getBasinData()
.getAverageValue(pfafs, recentTime);
} else {
value = getBasin(key, field, recentTime, aggregate)
.getValue(recentTime);
}
break;
case QPF: {
value = getBasin(key, field, recentTime, aggregate)
.getAverageValue(recentTime,
getQpfSourceExpiration());
if (aggregate) {
value = getQpfRecord(recentTime).getBasinData()
.getAverageValue(pfafs, recentTime,
getQpfSourceExpiration());
} else {
value = getBasin(key, field, recentTime, aggregate)
.getAverageValue(recentTime,
getQpfSourceExpiration());
}
break;
}
case GUIDANCE: {
value = getGuidanceValue(
(FFMPGuidanceBasin) getBasin(key, field,
recentTime, aggregate), recentTime,
getFFGName());
if (aggregate) {
getGuidanceRecord()
.getBasinData()
.getAverageGuidanceValue(
pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration(getFFGName()));
} else {
value = getGuidanceValue(
(FFMPGuidanceBasin) getBasin(key, field,
recentTime, aggregate), recentTime,
getFFGName());
}
break;
}
case QPE: {
value = getBasin(key, field, recentTime, aggregate)
.getAccumValue(
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
if (aggregate) {
value = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData()
.getPrimarySourceXML()
.isRate());
} else {
value = getBasin(key, field, recentTime, aggregate)
.getAccumValue(
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData()
.getPrimarySourceXML()
.isRate());
}
break;
}
}
@ -810,17 +837,15 @@ public class FFMPResource extends
boolean forced = forceResult.isForced();
if ((forcedPfafs.size() > 0) && forced) {
// Recalculate the guidance using the forced value(s)
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(ffgType),
new Float(value), forcedPfafs,
getGuidSourceExpiration(ffgType));
value = guidRecord.getBasinData().getAverageGuidanceValue(
pfafList, this.getGuidanceInterpolation(ffgType),
new Float(value), forcedPfafs,
getGuidSourceExpiration(ffgType));
} else if (forcedPfafs.size() > 0) {
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(ffgType),
Float.NaN, forcedPfafs,
getGuidSourceExpiration(ffgType));
value = guidRecord.getBasinData().getAverageGuidanceValue(
pfafList, this.getGuidanceInterpolation(ffgType),
Float.NaN, forcedPfafs,
getGuidSourceExpiration(ffgType));
}
}
@ -906,10 +931,8 @@ public class FFMPResource extends
if ((rateRecord == null) && isNewRate) {
try {
String huc = getHucIfWorstCase();
rateRecord = monitor.getRateRecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), recentTime, huc,
false);
getDataKey(), getPrimarySource(), recentTime, false);
isNewRate = false;
} catch (Exception e) {
e.printStackTrace();
@ -926,10 +949,9 @@ public class FFMPResource extends
public FFMPRecord getQpeRecord() {
try {
if ((qpeRecord == null) && (getTableTime() != null) && isNewQpe) {
String huc = getHucIfWorstCase();
qpeRecord = monitor.getQPERecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), getTableTime(), huc,
false);
qpeRecord = monitor
.getQPERecord(getProduct(), getSiteKey(), getDataKey(),
getPrimarySource(), getTableTime(), false);
isNewQpe = false;
}
} catch (Exception e) {
@ -992,9 +1014,8 @@ public class FFMPResource extends
}
}
String huc = getHucIfWorstCase();
qpfRecord = monitor.getQPFRecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), date, huc, false);
getDataKey(), getPrimarySource(), date, false);
isNewQpf = false;
}
} catch (Exception e) {
@ -1014,7 +1035,7 @@ public class FFMPResource extends
if ((virtualRecord == null) && isNewVirtual) {
virtualRecord = monitor.getVirtualRecord(getProduct(),
getSiteKey(), getDataKey(), getPrimarySource(),
getTableTime(), FFMPRecord.ALL, false);
getTableTime(), false);
isNewVirtual = false;
}
@ -1801,13 +1822,13 @@ public class FFMPResource extends
List<Float> guids = null;
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpes = getQpeRecord().getBasinData(FFMPRecord.ALL)
qpes = getQpeRecord().getBasinData()
.getAccumValues(pfafs, getTableTime(),
recentTime, getQpeSourceExpiration(),
isRate());
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getGuidanceValues(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
@ -1818,20 +1839,19 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(getHuc())
.get(key)
.getAccumValue(
qpe = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, ffgType);
guid = getGuidanceRecord().getBasinData()
.getAverageGuidanceValue(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
diff = FFMPUtils.getDiffValue(qpe, guid);
}
@ -1839,14 +1859,14 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.get(key)
.getAccumValue(getTableTime(), recentTime,
getQpeSourceExpiration(), isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
.getBasinData().get(key),
recentTime, ffgType);
guid = forceValue(pfafs,
getBasin(key, getField(), recentTime, aggregate),
@ -1881,14 +1901,14 @@ public class FFMPResource extends
List<Float> qpes = null;
List<Float> guids = null;
if (getQpeRecord() != null) {
qpes = getQpeRecord().getBasinData(FFMPRecord.ALL)
qpes = getQpeRecord().getBasinData()
.getAccumValues(pfafs, getTableTime(),
recentTime, getQpeSourceExpiration(),
isRate());
}
if (getGuidanceRecord() != null) {
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getGuidanceValues(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
@ -1899,32 +1919,31 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(getHuc())
.get(key)
.getAccumValue(
qpe = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
guid = getGuidanceRecord().getBasinData()
.getAverageGuidanceValue(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
}
}
} else {
if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.get(key)
.getAccumValue(getTableTime(), recentTime,
getQpeSourceExpiration(), isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
.getBasinData().get(key),
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
}
@ -4043,5 +4062,5 @@ public class FFMPResource extends
}
return dataTimes;
}
}

69
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java
View file

@ -32,7 +32,6 @@ import javax.xml.bind.annotation.XmlType;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.IJobChangeEvent;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.jobs.JobChangeAdapter;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
@ -81,6 +80,7 @@ import com.raytheon.uf.viz.monitor.ffmp.xml.FFMPConfigBasinXML;
* Mar 06, 2013 1769 dhladky Changed threading to use count down latch.
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen Use new load jobs
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -208,7 +208,6 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
}
}
}
this.domains = defaults;
final Date mostRecentTime = availableTimes[availableTimes.length - 1]
@ -216,56 +215,23 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
final double configTimeFrame = cfgBasinXML.getTimeFrame();
final Date timeBack = new Date(
(long) (mostRecentTime.getTime() - (configTimeFrame * TimeUtil.MILLIS_PER_HOUR)));
final List<String> initialHucs = new ArrayList<String>();
initialHucs.add(FFMPRecord.ALL);
final String defaultLayer = monitor.getConfig()
.getFFMPConfigData().getLayer();
if (!defaultLayer.equals(FFMPRecord.ALL)) {
initialHucs.add(defaultLayer);
}
final List<String> onlyAllHuc = new ArrayList<String>();
onlyAllHuc.add(FFMPRecord.ALL);
InitialLoadJob initialJob = new InitialLoadJob(this, timeBack,
mostRecentTime, initialHucs);
mostRecentTime, onlyAllHuc);
// schedule the secondary load to start as soon as the initial
// completes
// secondary load will be the same time period as initial with
// the hucs that the initial job did not do
// schedule the background load to start as soon as the initial
// completes, it will load the other hours back to 24
initialJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
Date secondStartTime = timeBack;
List<String> secondaryHucs = FFMPTemplateConfigurationManager
.getInstance().getHucLevels();
secondaryHucs.removeAll(initialHucs);
BackgroundLoadJob secondaryJob = new BackgroundLoadJob(
"Secondary FFMP Load", FFMPResourceData.this,
secondStartTime, mostRecentTime, secondaryHucs);
secondaryJob.setPriority(Job.SHORT);
// schedule the tertiary load as soon as the
// secondary completes
// tertiary load will do 24 hours back of the
// same hucs as the initial load
secondaryJob
.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
List<String> tertiaryHucs = new ArrayList<String>();
tertiaryHucs.add(FFMPRecord.ALL);
Date tertiaryStartTime = new Date(
mostRecentTime.getTime()
- (24 * TimeUtil.MILLIS_PER_HOUR));
BackgroundLoadJob tertiaryJob = new BackgroundLoadJob(
"Tertiary FFMP Load",
FFMPResourceData.this,
tertiaryStartTime, timeBack,
tertiaryHucs);
tertiaryJob
.setPreloadAvailableUris(true);
tertiaryJob.schedule();
}
});
secondaryJob.schedule();
Date backgroundStartTime = new Date(mostRecentTime
.getTime() - (24 * TimeUtil.MILLIS_PER_HOUR));
BackgroundLoadJob backgroundJob = new BackgroundLoadJob(
"Background FFMP Load", FFMPResourceData.this,
backgroundStartTime, timeBack, onlyAllHuc);
backgroundJob.setPreloadAvailableUris(true);
backgroundJob.schedule();
}
});
initialJob.schedule();
@ -274,7 +240,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
// background so the first paints of the resource
// will be faster
List<String> earlyLoadHucs = new ArrayList<String>();
earlyLoadHucs.addAll(initialHucs);
earlyLoadHucs.addAll(onlyAllHuc);
for (String otherHuc : FFMPTemplateConfigurationManager
.getInstance().getHucLevels()) {
if (!earlyLoadHucs.contains(otherHuc)) {
@ -324,8 +290,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
.getAvailableUris(siteKey, dataKey, sourceName,
standAloneTime);
monitor.processUris(sourceURIs, siteKey, sourceName,
standAloneTime, FFMPRecord.ALL,
SubMonitor.convert(null));
standAloneTime, SubMonitor.convert(null));
}
}
}
@ -427,10 +392,10 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
public void populateRecord(FFMPRecord precord) throws VizException {
try {
getMonitor().populateFFMPRecord(siteKey, precord,
precord.getSourceName(), huc);
precord.getSourceName());
} catch (Exception e) {
throw new VizException("Failed to populate ffmp record "
+ precord.getDataURI() + " for huc " + huc);
+ precord.getDataURI());
}
}

188
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java
View file

@ -59,6 +59,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 11, 2013 2085 njensen Initial creation
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -208,7 +209,7 @@ public class FFMPRowGenerator implements Runnable {
new FFMPTableCellData(rowField, sb.toString(),
mouseOverText));
if (!isWorstCase || huc.equals(FFMPRecord.ALL)
if (huc.equals(FFMPRecord.ALL)
|| (centeredAggregationKey != null)) {
if (!cBasin.getValues().isEmpty()) {
@ -272,8 +273,73 @@ public class FFMPRowGenerator implements Runnable {
FIELDS.DIFF, diffValue));
i += 3;
}
} else {
} else if (isWorstCase) {
trd = getMaxValue(trd, cBasin);
} else {
// general Aggregate HUC processing
ArrayList<Long> pfafs = ft.getAggregatePfafs(
cBasin.getPfaf(), siteKey, domain);
if (!cBasin.getValues().isEmpty()) {
rate = vgBasin.getAverageValue(paintRefTime,
expirationTime);
if (sliderTime > 0.00) {
FFMPTimeWindow window = monitor.getQpeWindow();
qpeBasin.getAccumAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
} else {
qpe = 0.0f;
}
}
trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE,
rate));
trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE,
qpe));
if (qpfBasin != null) {
FFMPTimeWindow window = monitor.getQpfWindow();
qpf = qpfBasin.getAverageValue(pfafs,
window.getAfterTime(), window.getBeforeTime());
}
trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF,
qpf));
// run over each guidance type
int i = 0;
for (String guidType : guidBasins.keySet()) {
guidance = Float.NaN;
FFMPTableCellData guidCellData = getGuidanceCellData(
cBasin, domain, guidType, parentBasinPfaf);
if (guidCellData == null) {
// check for forcing even if no data are available
guidance = getForcedAvg(domain, cBasin, guidType);
boolean forced = !guidance.isNaN();
guidCellData = new FFMPTableCellData(
FIELDS.GUIDANCE, guidance, forced);
} else {
guidance = guidCellData.getValueAsFloat();
}
trd.setTableCellData(i + 4, guidCellData);
float ratioValue = Float.NaN;
float diffValue = Float.NaN;
// If guidance is NaN then it cannot be > 0
if (!qpe.isNaN() && (guidance > 0.0f)) {
ratioValue = FFMPUtils.getRatioValue(qpe, guidance);
diffValue = FFMPUtils.getDiffValue(qpe, guidance);
}
trd.setTableCellData(i + 5, new FFMPTableCellData(
FIELDS.RATIO, ratioValue));
trd.setTableCellData(i + 6, new FFMPTableCellData(
FIELDS.DIFF, diffValue));
i += 3;
}
}
trd.setSortCallback(tData);
@ -292,22 +358,46 @@ public class FFMPRowGenerator implements Runnable {
if (!isWorstCase || huc.equals(FFMPRecord.ALL)
|| (centeredAggregationKey != null)) {
ArrayList<Long> pfafs = null;
if (cBasin.getAggregated()) {
pfafs = ft.getAggregatePfafs(cBasin.getPfaf(), siteKey,
huc);
}
if (rateBasin != null) {
FFMPBasin basin = rateBasin.get(cBasinPfaf);
if (basin != null) {
rate = basin.getValue(paintRefTime);
if (cBasin.getAggregated()) {
rate = rateBasin.getAverageValue(pfafs,
paintRefTime);
} else {
FFMPBasin basin = rateBasin.get(cBasinPfaf);
if (basin != null) {
rate = basin.getValue(paintRefTime);
}
}
}
trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE,
rate));
if (qpeBasin != null) {
FFMPBasin basin = qpeBasin.get(cBasinPfaf);
if (basin != null) {
FFMPTimeWindow window = monitor.getQpeWindow();
qpe = basin.getAccumValue(window.getAfterTime(),
FFMPTimeWindow window = monitor.getQpeWindow();
if (cBasin.getAggregated()) {
qpe = qpeBasin.getAccumAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
} else {
FFMPBasin basin = qpeBasin.get(cBasinPfaf);
if (basin != null) {
qpe = basin.getAccumValue(
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
}
}
}
@ -315,11 +405,19 @@ public class FFMPRowGenerator implements Runnable {
qpe));
if (qpfBasin != null) {
FFMPBasin basin = qpfBasin.get(cBasinPfaf);
if (basin != null) {
FFMPTimeWindow window = monitor.getQpfWindow();
qpf = basin.getAverageValue(window.getAfterTime(),
FFMPTimeWindow window = monitor.getQpfWindow();
if (cBasin.getAggregated()) {
qpf = qpfBasin.getAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime());
} else {
FFMPBasin basin = qpfBasin.get(cBasinPfaf);
if (basin != null) {
qpf = basin.getAverageValue(
window.getAfterTime(),
window.getBeforeTime());
}
}
}
@ -436,13 +534,13 @@ public class FFMPRowGenerator implements Runnable {
if (cBasin instanceof FFMPVirtualGageBasin) {
if (!pfafs.isEmpty()) {
if (virtualBasin != null) {
rate = virtualBasin.get(cBasin.getPfaf()).getValue(
paintRefTime);
rate = virtualBasin.getMaxValue(pfafs, paintRefTime);
if (sliderTime > 0.00) {
qpe = virtualBasin.get(cBasin.getPfaf()).getAccumValue(
monitor.getQpeWindow().getAfterTime(),
monitor.getQpeWindow().getBeforeTime(),
qpe = virtualBasin.getAccumMaxValue(pfafs, monitor
.getQpeWindow().getAfterTime(), monitor
.getQpeWindow().getBeforeTime(),
expirationTime, isRate);
} else {
qpe = 0.0f;
@ -454,10 +552,9 @@ public class FFMPRowGenerator implements Runnable {
trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE, qpe));
if (qpfBasin != null) {
qpf = new Float(qpfBasin.get(cBasin.getPfaf()).getMaxValue(
monitor.getQpfWindow().getAfterTime(),
monitor.getQpfWindow().getBeforeTime()))
.floatValue();
qpf = qpfBasin.getMaxValue(pfafs, monitor.getQpfWindow()
.getAfterTime(), monitor.getQpfWindow()
.getBeforeTime());
}
trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF, qpf));
@ -534,7 +631,7 @@ public class FFMPRowGenerator implements Runnable {
if (isWorstCase) {
guidance = guidRecords
.get(guidType)
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getMaxGuidanceValue(
pfafs,
resource.getGuidanceInterpolators()
@ -542,11 +639,16 @@ public class FFMPRowGenerator implements Runnable {
resource.getGuidSourceExpiration(guidType),
cBasin.getPfaf());
} else {
FFMPGuidanceBasin basin = (FFMPGuidanceBasin) guidRecords
.get(guidType).getBasinData(huc)
.get(cBasin.getPfaf());
guidance = resource.getGuidanceValue(basin, monitor
.getQpeWindow().getBeforeTime(), guidType);
guidance = guidRecords
.get(guidType)
.getBasinData()
.getAverageGuidanceValue(
pfafs,
resource.getGuidanceInterpolators()
.get(guidType),
resource.getGuidSourceExpiration(guidType));
}
trd.setTableCellData(i + 4, new FFMPTableCellData(
@ -766,17 +868,6 @@ public class FFMPRowGenerator implements Runnable {
private FFMPTableCellData getGuidanceCellData(FFMPBasin cBasin,
String domain, String guidType, Long parentBasinPfaf) {
long cBasinPfaf = cBasin.getPfaf();
FFMPBasinData guidBasin = guidBasins.get(guidType);
FFMPGuidanceBasin ffmpGuidBasin = null;
if (guidBasin != null) {
ffmpGuidBasin = (FFMPGuidanceBasin) guidBasin.get(cBasinPfaf);
}
if (ffmpGuidBasin == null) {
return null;
}
List<Long> pfafList = Collections.emptyList();
List<Long> forcedPfafs = Collections.emptyList();
boolean forced = false;
@ -806,8 +897,7 @@ public class FFMPRowGenerator implements Runnable {
if (FFFGDataMgr.getInstance().isForcingConfigured()) {
FFMPBasin parentBasin = cBasin;
if (cBasinPfaf != parentBasinPfaf.longValue()) {
parentBasin = baseRec.getBasinData(FFMPRecord.ALL).get(
parentBasinPfaf);
parentBasin = baseRec.getBasinData().get(parentBasinPfaf);
}
ForceUtilResult forceResult = forceUtil.calculateForcings(domain,
ft, parentBasin);
@ -819,20 +909,22 @@ public class FFMPRowGenerator implements Runnable {
// Recalculate guidance using the forced value(s)
guidance = guidRecords
.get(guidType)
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getAverageGuidanceValue(pfafList,
resource.getGuidanceInterpolators().get(guidType),
guidance, forcedPfafs,
resource.getGuidSourceExpiration(guidType));
} else {
if (ffmpGuidBasin != null) {
guidance = resource.getGuidanceValue(ffmpGuidBasin,
paintRefTime, guidType);
if (guidance < 0.0f) {
guidance = Float.NaN;
}
FFMPGuidanceBasin ffmpGuidBasin = (FFMPGuidanceBasin) guidRecords
.get(guidType).getBasinData().get(cBasinPfaf);
guidance = resource.getGuidanceValue(ffmpGuidBasin, paintRefTime,
guidType);
if (guidance < 0.0f) {
guidance = Float.NaN;
}
}
return new FFMPTableCellData(FIELDS.GUIDANCE, guidance, forced);

51
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java
View file

@ -22,7 +22,6 @@ package com.raytheon.uf.viz.monitor.ffmp.ui.thread;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@ -31,7 +30,6 @@ import java.util.Set;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.Job;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.xml.FFMPRunXML;
import com.raytheon.uf.common.monitor.xml.ProductRunXML;
@ -54,6 +52,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.rsc.FFMPResourceData;
* ------------ ---------- ----------- --------------------------
* Jun 04, 2013 2075 njensen Initial creation
* Jun 07, 2013 2075 njensen Added progress monitoring
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -81,8 +80,6 @@ public abstract class AbstractLoadJob extends Job {
protected FFMPMonitor ffmpMonitor;
protected List<String> hucsToLoad = null;
/**
* Constructor
*
@ -105,28 +102,13 @@ public abstract class AbstractLoadJob extends Job {
this.resourceData = resourceData;
this.startTime = timeBack;
this.endTime = mostRecentTime;
this.hucsToLoad = hucsToLoad;
// configure FFMP
this.hucsToLoad.remove(FFMPRecord.VIRTUAL);
FFMPRunXML runXML = FFMPRunConfigurationManager.getInstance()
.getRunner(resourceData.wfo);
this.productRun = runXML.getProduct(resourceData.siteKey);
this.product = resourceData.getProduct();
this.ffmpMonitor = FFMPMonitor.getInstance();
// just for debugging/logging
StringBuilder sb = new StringBuilder();
sb.append(name);
sb.append(" hucs to load: ");
Iterator<String> itr = this.hucsToLoad.iterator();
while (itr.hasNext()) {
sb.append(itr.next());
if (itr.hasNext()) {
sb.append(", ");
}
}
System.out.println(sb.toString());
}
/**
@ -165,10 +147,8 @@ public abstract class AbstractLoadJob extends Job {
}
}
if (rateURI != null) {
for (String phuc : hucsToLoad) {
ffmpMonitor.processUri(rateURI, resourceData.siteKey,
product.getRate(), startTime, phuc);
}
ffmpMonitor.processUri(rateURI, resourceData.siteKey,
product.getRate(), startTime);
}
}
@ -190,12 +170,10 @@ public abstract class AbstractLoadJob extends Job {
protected void doQpe(NavigableMap<Date, List<String>> qpeURIs,
SubMonitor smonitor) {
if (!qpeURIs.isEmpty()) {
smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR);
for (String phuc : hucsToLoad) {
ffmpMonitor.processUris(qpeURIs, resourceData.siteKey,
product.getQpe(), startTime, phuc,
smonitor.newChild(PROGRESS_FACTOR));
}
smonitor.beginTask(null, PROGRESS_FACTOR);
ffmpMonitor.processUris(qpeURIs, resourceData.siteKey,
product.getQpe(), startTime,
smonitor.newChild(PROGRESS_FACTOR));
}
}
@ -246,12 +224,11 @@ public abstract class AbstractLoadJob extends Job {
// Use this method of QPF data retrieval if you don't have cache
// files
if (!qpfURIs.isEmpty()) {
smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR);
for (String phuc : hucsToLoad) {
ffmpMonitor.processUris(qpfURIs, resourceData.siteKey,
productQpf, startTime, phuc,
smonitor.newChild(PROGRESS_FACTOR));
}
smonitor.beginTask(null, PROGRESS_FACTOR);
ffmpMonitor.processUris(qpfURIs, resourceData.siteKey, productQpf,
startTime, smonitor.newChild(PROGRESS_FACTOR));
}
}
@ -264,7 +241,7 @@ public abstract class AbstractLoadJob extends Job {
product.getVirtual(), startTime);
if (!virtualURIs.isEmpty()) {
ffmpMonitor.processUris(virtualURIs, resourceData.siteKey,
product.getVirtual(), startTime, FFMPRecord.ALL, smonitor);
product.getVirtual(), startTime, smonitor);
}
}
@ -303,7 +280,7 @@ public abstract class AbstractLoadJob extends Job {
if (iguidURIs != null && !iguidURIs.isEmpty()) {
ffmpMonitor.processUris(iguidURIs, resourceData.siteKey,
guidSource.getSourceName(), startTime,
FFMPRecord.ALL,
smonitor.newChild(PROGRESS_FACTOR / subWork));
}
}

42
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java
View file

@ -21,7 +21,6 @@ package com.raytheon.uf.common.dataplugin.ffmp;
**/
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.raytheon.uf.common.serialization.ISerializableObject;
@ -37,6 +36,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Created to reduce memory and disk read/writes for FFMP
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -52,8 +52,6 @@ public class FFMPAggregateRecord implements ISerializableObject {
@DynamicSerializeElement
private String sourceSiteDataKey;
private static final long serialVersionUID = 767745643535471L;
/**
* Aggregate Record implementation for FFMP
@ -63,8 +61,16 @@ public class FFMPAggregateRecord implements ISerializableObject {
}
@DynamicSerializeElement
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
private FFMPBasinData basins;
public FFMPBasinData getBasins() {
return basins;
}
public void setBasins(FFMPBasinData basins) {
this.basins = basins;
}
@DynamicSerializeElement
private List<Long> times = new ArrayList<Long>();
@ -83,34 +89,6 @@ public class FFMPAggregateRecord implements ISerializableObject {
public List<Long> getTimes() {
return times;
}
public void setBasinsMap(HashMap<String, FFMPBasinData> basinsMap) {
this.basinsMap = basinsMap;
}
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Add a basin Data aggregate object
* @param cacheData
*/
public void addBasinData(FFMPBasinData basinData) {
basinsMap.put(basinData.getHucLevel(), basinData);
}
/**
* Gets the BasinData object
* @param huc
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinsMap.containsKey(huc)) {
return basinsMap.get(huc);
}
return null;
}
/**
* WFO designator

2
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java
View file

@ -31,7 +31,6 @@ import javax.persistence.Transient;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.time.util.ImmutableDate;
/**
* FFMP basin/aggregated value holder
@ -48,6 +47,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* from aggregate records and delayed
* TreeMap creation to the tertiary loader.
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*

94
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java
View file

@ -51,6 +51,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* 01/27/13 1569 D. Hladky Added support for write of aggregate record cache
* 04/16/13 1912 bsteffen Initial bulk hdf5 access for ffmp
* 05/09/13 1919 mpduff Use parent pfaf instead of lookupId.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -59,7 +60,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
*/
@DynamicSerialize
public class FFMPBasinData implements ISerializableObject {
private static final long serialVersionUID = 8162247989509750715L;
public static final double GUIDANCE_MISSING = -999999.0;
@ -80,15 +80,6 @@ public class FFMPBasinData implements ISerializableObject {
*/
private final Map<String, FFMPBasin[]> orderedBasinsCache = new HashMap<String, FFMPBasin[]>();
/**
* Public one arg constructor
*
* @param huc_level
*/
public FFMPBasinData(String hucLevel) {
setHucLevel(hucLevel);
}
/**
* No arg hibernate constructor
*/
@ -172,6 +163,8 @@ public class FFMPBasinData implements ISerializableObject {
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param startDate
* @param finishDate
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date beforeDate,
@ -191,6 +184,55 @@ public class FFMPBasinData implements ISerializableObject {
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param exact
* date
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date date) {
float tvalue = 0.0f;
int i = 0;
for (Long pfaf : pfaf_ids) {
FFMPBasin basin = getBasins().get(pfaf);
if (basin != null) {
tvalue += basin.getValue(date);
i++;
}
}
tvalue = tvalue / i;
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param date
* @param expirationTime
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date date,
long epirationTime) {
float tvalue = 0.0f;
int i = 0;
for (Long pfaf : pfaf_ids) {
FFMPBasin basin = getBasins().get(pfaf);
if (basin != null) {
tvalue += basin.getAverageValue(date, epirationTime);
i++;
}
}
tvalue = tvalue / i;
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins with areal
* weighting
@ -229,8 +271,8 @@ public class FFMPBasinData implements ISerializableObject {
* @param pfaf_ids
* @return
*/
public float getAccumAverageValue(ArrayList<Long> pfaf_ids,
Date beforeDate, Date afterDate, long expirationTime, boolean rate) {
public float getAccumAverageValue(List<Long> pfaf_ids, Date beforeDate,
Date afterDate, long expirationTime, boolean rate) {
float tvalue = 0.0f;
int i = 0;
@ -534,6 +576,34 @@ public class FFMPBasinData implements ISerializableObject {
return values;
}
/**
* Gets the average guidance value for an aggregate basin
*
* @param pfaf_ids
* @param interpolation
* @param expiration
* @return
*/
public Float getAverageGuidanceValue(List<Long> pfaf_ids,
FFMPGuidanceInterpolation interpolation, long expiration) {
float tvalue = 0.0f;
int i = 0;
List<Float> vals = getGuidanceValues(pfaf_ids, interpolation,
expiration);
if (vals != null) {
for (Float val : vals) {
tvalue += val;
i++;
}
} else {
return null;
}
return tvalue / i;
}
/**
* used for max ratio and diff calcs
*

162
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java
View file

@ -24,8 +24,6 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.SOURCE_TYPE;
import com.raytheon.uf.common.monitor.xml.SourceXML;
@ -47,6 +45,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException
* 01/27/13 1478 D. Hladky Re-worked to help with memory size and NAS read write stress
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -59,10 +58,10 @@ public class FFMPDataContainer {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPDataContainer.class);
private final Map<String, FFMPBasinData> basinDataMap = new ConcurrentHashMap<String, FFMPBasinData>();// DR
private FFMPBasinData basins;
private String sourceName = null;
private boolean isPurged = false;
public FFMPDataContainer() {
@ -71,40 +70,41 @@ public class FFMPDataContainer {
/**
* Usual constructor
*
* @param sourceName
*/
public FFMPDataContainer(String sourceName) {
this.sourceName = sourceName;
basinDataMap.put(FFMPRecord.ALL, new FFMPBasinData(FFMPRecord.ALL));
// System.out.println("Creating source: " + sourceName);
basins = new FFMPBasinData();
}
/**
* special constuctor
*
* @param sourceName
* @param hucs
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
basinDataMap.put(huc, new FFMPBasinData(huc));
}
basins = new FFMPBasinData();
}
/**
* new container first time read in from cache
*
* @param sourceName
* @param hucs
* @param record
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs, FFMPAggregateRecord record) {
public FFMPDataContainer(String sourceName, FFMPAggregateRecord record) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
FFMPBasinData basinData = record.getBasinData(huc);
basinData.populate(record.getTimes());
basinDataMap.put(huc, basinData);
}
FFMPBasinData basinData = record.getBasins();
basinData.populate(record.getTimes());
basins = basinData;
}
/**
@ -114,7 +114,7 @@ public class FFMPDataContainer {
* @param newBasinData
*/
public void addFFMPEntry(Date date, SourceXML source,
FFMPBasinData newBasinData, String huc, String siteKey) {
FFMPBasinData newBasinData, String siteKey) {
boolean guid = false;
@ -122,12 +122,12 @@ public class FFMPDataContainer {
guid = true;
}
FFMPBasinData currBasinData = getBasinData(huc);
FFMPBasinData currBasinData = getBasinData();
synchronized (currBasinData) {
if (currBasinData == null) {
setBasinData(huc, newBasinData);
setBasinData(newBasinData);
} else {
for (Long key : newBasinData.getBasins().keySet()) {
@ -158,7 +158,7 @@ public class FFMPDataContainer {
val);
}
//currBasinData.put(key, basin);
// currBasinData.put(key, basin);
syncPut(currBasinData, key, basin);
} else {
@ -243,7 +243,7 @@ public class FFMPDataContainer {
}
basin.setValue(date, val);
//currBasinData.put(key, basin);
// currBasinData.put(key, basin);
syncPut(currBasinData, key, basin);
} else {
@ -283,10 +283,9 @@ public class FFMPDataContainer {
public boolean containsKey(Date date) {
boolean contains = false;
if (getBasinData(FFMPRecord.ALL) != null) {
if (getBasinData() != null) {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -309,7 +308,7 @@ public class FFMPDataContainer {
*/
public boolean containsKey(String sourceName) {
boolean contains = false;
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL).getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -334,18 +333,14 @@ public class FFMPDataContainer {
*
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinDataMap.containsKey(huc)) {
return basinDataMap.get(huc);
public FFMPBasinData getBasinData() {
if (basins != null) {
return basins;
} else {
return null;
}
}
public Set<String> getKeys() {
return basinDataMap.keySet();
}
/**
* Get the maximum value in the monitored area.
*
@ -359,9 +354,8 @@ public class FFMPDataContainer {
public double getMaxValue(ArrayList<Long> pfafs, Date backDate,
Date currDate, long expirationTime, boolean rate) {
double val = getBasinData(FFMPRecord.ALL).getAccumMaxValue(pfafs,
currDate,
backDate, expirationTime, rate);
double val = getBasinData().getAccumMaxValue(pfafs, currDate, backDate,
expirationTime, rate);
return val;
}
@ -374,8 +368,7 @@ public class FFMPDataContainer {
public Date getNewest() {
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -402,8 +395,7 @@ public class FFMPDataContainer {
*/
public Date getOldest() {
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -431,8 +423,7 @@ public class FFMPDataContainer {
public List<Date> getOrderedTimes(Date barrierTime) {
ArrayList<Date> orderedTimes = new ArrayList<Date>();
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -453,16 +444,16 @@ public class FFMPDataContainer {
return null;
}
/**
* Gets the list of times for serialization
*
* @return
*/
public List<Long> getOrderedTimes() {
ArrayList<Long> orderedTimes = new ArrayList<Long>();
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -484,6 +475,7 @@ public class FFMPDataContainer {
/**
* Gets the source name for this Data Container
*
* @return
*/
public String getSourceName() {
@ -492,13 +484,13 @@ public class FFMPDataContainer {
/**
* Clean up old junk
*
* @param backDate
*/
public void purge(Date backDate) {
statusHandler.handle(Priority.INFO, "Purging "+getSourceName()+" Container back to: "+backDate);
for (String huc : basinDataMap.keySet()) {
getBasinData(huc).purgeData(backDate);
}
statusHandler.handle(Priority.INFO, "Purging " + getSourceName()
+ " Container back to: " + backDate);
getBasinData().purgeData(backDate);
setPurged(true);
}
@ -511,27 +503,25 @@ public class FFMPDataContainer {
// create a record from the cache record
FFMPRecord record = new FFMPRecord(cacheRecord);
for (Entry<String, FFMPBasinData> dentry : record.getBasinsMap()
.entrySet()) {
for (Entry<Long, FFMPBasin> entry : dentry.getValue().getBasins()
.entrySet()) {
FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
basin.getValues().putAll(entry.getValue().getValues());
}
FFMPBasinData dentry = record.getBasinData();
for (Entry<Long, FFMPBasin> entry : dentry.getBasins().entrySet()) {
FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
syncPut(getBasinData(dentry.getKey()), entry.getKey(),
entry.getValue());
basin.getValues().putAll(entry.getValue().getValues());
}
} else {
syncPut(getBasinData(), entry.getKey(), entry.getValue());
}
}
}
/**
@ -539,8 +529,8 @@ public class FFMPDataContainer {
*
* @param basins
*/
public void setBasinData(String huc, FFMPBasinData fftiData) {
basinDataMap.put(huc, fftiData);
public void setBasinData(FFMPBasinData basins) {
this.basins = basins;
}
public void setSourceName(String sourceName) {
@ -555,7 +545,7 @@ public class FFMPDataContainer {
*/
public int size() {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL).getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -569,34 +559,27 @@ public class FFMPDataContainer {
}
return 0;
}
/**
* DR 15471 lock put() to avoid ConcurrentModificationException
*/
private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value){
if(fbd==null || key==null)
return;
Map<Long, FFMPBasin> basins = fbd.getBasins();
if(basins == null)
return;
synchronized (basins) {
basins.put(key, value);
}
}
/**
* Gets the basin data map
* @return
* DR 15471 lock put() to avoid ConcurrentModificationException
*/
public Map<String, FFMPBasinData> getBasinMap() {
return basinDataMap;
private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value) {
if (fbd == null || key == null)
return;
Map<Long, FFMPBasin> basins = fbd.getBasins();
if (basins == null)
return;
synchronized (basins) {
basins.put(key, value);
}
}
/**
* Sets whether this container has been purged or not
*
* @param isPurged
*/
public void setPurged(boolean isPurged) {
@ -605,6 +588,7 @@ public class FFMPDataContainer {
/**
* Has this container been purged?
*
* @return
*/
public boolean isPurged() {

115
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java
View file

@ -24,7 +24,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.lang.ref.WeakReference;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@ -89,6 +88,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* Apr 18, 2013 1919 dhladky Added method for VGB loading
* May 07, 2013 1869 bsteffen Remove dataURI column from
* PluginDataObject.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -142,23 +142,25 @@ public class FFMPRecord extends PersistablePluginDataObject
private String siteKey;
@Transient
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
private FFMPBasinData basins = new FFMPBasinData();;
@Transient
private int expiration = 0;
@Transient
private boolean isRate = false;
protected static ConcurrentMap<Long, WeakReference<ImmutableDate>> cacheTimes = new ConcurrentHashMap<Long, WeakReference<ImmutableDate>>();
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPRecord.class);
/** ALL HUC LEVEL **/
public static final String ALL = "ALL";
/** COUNTY HUC LEVEL **/
public static final String COUNTY = "COUNTY";
/** VIRTUAL HUC LEVEL **/
public static final String VIRTUAL = "VIRTUAL";
@ -359,8 +361,8 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param hucName
*/
public void setBasinData(FFMPBasinData basins, String hucName) {
basinsMap.put(hucName, basins);
public void setBasinData(FFMPBasinData basins) {
this.basins = basins;
}
/**
@ -369,24 +371,10 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param basins
* @param hucName
*/
public FFMPBasinData getBasinData(String hucName) {
FFMPBasinData basins = basinsMap.get(hucName);
if (basins == null) {
basins = new FFMPBasinData(hucName);
basinsMap.put(hucName, basins);
}
public FFMPBasinData getBasinData() {
return basins;
}
/**
* Gets the map if you need it
*
* @return
*/
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Gets the Hash out of the datastore by HUC
*
@ -394,41 +382,36 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param huc
*/
public void retrieveMapFromDataStore(File datastoreFile, String uri,
FFMPTemplates template, String huc, Date date, String sourceName)
FFMPTemplates template, Date date, String sourceName)
throws Exception {
FFMPBasinData fbd = getBasinData(huc);
FFMPBasinData fbd = getBasinData();
ImmutableDate idate = getCacheDate(date);
boolean aggregate = true;
if (huc.equals(ALL)) {
aggregate = false;
}
boolean aggregate = false;
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(getSiteKey(),
domain.getCwa(), huc);
domain.getCwa(), FFMPRecord.ALL);
if (map != null && !map.isEmpty()) {
fbd.addBasins(datastoreFile, uri, getSiteKey(),
domain.getCwa(), huc, sourceName, idate, map.keySet(),
aggregate);
domain.getCwa(), FFMPRecord.ALL, sourceName, idate,
map.keySet(), aggregate);
}
}
}
public void retrieveMapFromDataStore(FFMPTemplates template, String huc)
public void retrieveMapFromDataStore(FFMPTemplates template)
throws Exception {
retrieveMapFromDataStore(getDataStoreFile(), getDataURI(), template,
huc, getDataTime().getRefTime(), getSourceName());
getDataTime().getRefTime(), getSourceName());
}
public void retrieveVirtualMapFromDataStore(FFMPTemplates template,
String huc) throws Exception {
retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(), template,
getDataTime().getRefTime(), getSourceName());
public void retrieveVirtualMapFromDataStore(FFMPTemplates template)
throws Exception {
retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(),
template, getDataTime().getRefTime(), getSourceName());
}
private File getDataStoreFile() {
@ -523,7 +506,7 @@ public class FFMPRecord extends PersistablePluginDataObject
public void retrieveVirtualMapFromDataStore(File datastoreFile, String uri,
FFMPTemplates template, Date date, String sourceName)
throws StorageException, FileNotFoundException {
FFMPBasinData fbd = getBasinData(ALL);
FFMPBasinData fbd = getBasinData();
String key = getDataKey();
ImmutableDate idate = getCacheDate(date);
@ -537,8 +520,7 @@ public class FFMPRecord extends PersistablePluginDataObject
if (size > 0) {
fbd.addVirtualBasins(datastoreFile, uri, key,
domain.getCwa(), idate,
lids.values());
domain.getCwa(), idate, lids.values());
}
}
}
@ -569,10 +551,9 @@ public class FFMPRecord extends PersistablePluginDataObject
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("\n dataURI: " + getDataURI() + "\n");
if (basinsMap != null) {
for (String key : basinsMap.keySet()) {
sb.append(key + " : " + basinsMap.get(key).getBasins().size()
+ "\n");
if (basins != null) {
for (Long key : basins.getBasins().keySet()) {
sb.append(key + " : " + basins.get(key).getValue() + "\n");
}
}
@ -635,10 +616,7 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param date
*/
public void purgeData(Date date) {
for (FFMPBasinData basinData : getBasinsMap().values()) {
basinData.purgeData(date);
}
basins.purgeData(date);
}
public void setSiteKey(String siteKey) {
@ -648,21 +626,20 @@ public class FFMPRecord extends PersistablePluginDataObject
public String getSiteKey() {
return siteKey;
}
/**
* Get the fully populated aggregate record
*
* @return
*/
public FFMPAggregateRecord getAggregateRecord() {
FFMPAggregateRecord fdcr = new FFMPAggregateRecord();
for (FFMPBasinData basinData: basinsMap.values()) {
fdcr.addBasinData(basinData);
}
fdcr.setBasins(basins);
return fdcr;
}
/**
* Creates and populates a version of this record from an aggregate record
*
@ -671,14 +648,11 @@ public class FFMPRecord extends PersistablePluginDataObject
public FFMPRecord(FFMPAggregateRecord fdcr) {
List<Long> times = fdcr.getTimes();
for (FFMPBasinData basinData: fdcr.getBasinsMap().values()) {
// Keep in mind times can be null, Guidance basins are like that
basinData.populate(times);
setBasinData(basinData, basinData.getHucLevel());
}
FFMPBasinData fdcrBasins = fdcr.getBasins();
fdcrBasins.populate(times);
setBasinData(fdcrBasins);
}
/**
* Gets and maintains the list of times. This will lesson memory consumption
* because it means all FFMPBasin TreeMap date keys reference back to this
@ -691,7 +665,7 @@ public class FFMPRecord extends PersistablePluginDataObject
WeakReference<ImmutableDate> idate = cacheTimes.get(date.getTime());
ImmutableDate myDate = null;
if (idate != null) {
myDate = idate.get();
}
@ -701,22 +675,19 @@ public class FFMPRecord extends PersistablePluginDataObject
myDate = new ImmutableDate(time);
idate = new WeakReference<ImmutableDate>(myDate);
cacheTimes.putIfAbsent(time, idate);
}
}
return myDate;
}
/**
* Populate data from the cache files
*
* @param basins
* @param hucName
*/
public void populate(FFMPBasinData basins, String hucName) {
setBasinData(basins, hucName);
//System.out.println("Adding Whole Object Cache Data: "+hucName+" "+getSourceName());
public void populate(FFMPBasinData basins) {
setBasinData(basins);
}
@Override

48
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java
View file

@ -88,6 +88,7 @@ import com.vividsolutions.jts.io.WKBReader;
* 03/18/13 1817 D. Hladky Fixed issue with BOX where only 1 HUC was showing up.
* 04/15/13 1902 M. Duff Generic List
* 06/10/13 2085 njensen Use countyMap for efficiency
* 07/15/13 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -1310,6 +1311,25 @@ public class FFMPTemplates {
return map;
}
/**
* Find the list of pfafs for this HUC level
* @param siteKey
* @param huc
* @param domains
* @return
*/
public synchronized List<Long> getHucKeyList(String siteKey, String huc, List<DomainXML> domains) {
Set<Long> keys = new HashSet<Long>();
for (DomainXML domain: domains) {
LinkedHashMap<Long, ?> map = getMap(siteKey, domain.getCwa(), huc);
keys.addAll(map.keySet());
}
return new ArrayList<Long>(keys);
}
/**
* Gets the template config manager
@ -1663,23 +1683,28 @@ public class FFMPTemplates {
}
HashMap<String, HashMap<Long, ArrayList<FFMPVirtualGageBasinMetaData>>> virtualMap = virtualGageBasinsInParentPfaf
.get(dataKey);
ArrayList<Long> result = new ArrayList<Long>();
for (DomainXML domain : domains) {
HashMap<Long, ArrayList<FFMPVirtualGageBasinMetaData>> map = virtualMap
.get(domain.getCwa());
if (map != null) {
ArrayList<FFMPVirtualGageBasinMetaData> list = map.get(pfaf);
if (list != null && !list.isEmpty()) {
ArrayList<Long> result = new ArrayList<Long>();
for (FFMPVirtualGageBasinMetaData md : list)
result.add(md.getLookupId());
return result;
for (FFMPVirtualGageBasinMetaData md : list) {
if (!result.contains(md.getLookupId())) {
result.add(md.getLookupId());
}
}
}
}
}
return new ArrayList<Long>();
return result;
}
@ -2423,6 +2448,8 @@ public class FFMPTemplates {
HashMap<String, HashMap<String, ArrayList<FFMPVirtualGageBasinMetaData>>> virtualMap = vgbsInCounty
.get(dataKey);
ArrayList<Long> result = new ArrayList<Long>();
for (DomainXML domain : domains) {
HashMap<String, ArrayList<FFMPVirtualGageBasinMetaData>> map = virtualMap
@ -2432,16 +2459,17 @@ public class FFMPTemplates {
.get(stateCommaCnty.trim().toUpperCase());
if (list != null && !list.isEmpty()) {
ArrayList<Long> result = new ArrayList<Long>();
for (FFMPVirtualGageBasinMetaData md : list) {
result.add(md.getLookupId());
for (FFMPVirtualGageBasinMetaData md : list) {
if (!result.contains(md.getLookupId())) {
result.add(md.getLookupId());
}
}
return result;
}
}
}
return new ArrayList<Long>();
return result;
}
}

144
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java
View file

@ -53,6 +53,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 07/01/09 2521 dhladky Initial Creation
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -103,63 +104,36 @@ public class FFMPDao extends PluginDao {
// ignore data outside of domain
if (vmap.size() > 0) {
for (String key : record.getBasinsMap().keySet()) {
FFMPBasinData fbd = record.getBasinData(key);
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(), key);
int size = 0;
if (key.equals(FFMPRecord.ALL)) {
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
size++;
}
}
} else {
for (Long pfaf : map.keySet()) {
ArrayList<Long> vgbpfafs = template
.getVGBsInAggregate(pfaf,
record.getSiteKey(), key);
if (vgbpfafs != null) {
size++;
}
}
FFMPBasinData fbd = record.getBasinData();
int size = 0;
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
size++;
}
float[] dataRec = new float[size];
int i = 0;
if (key.equals(FFMPRecord.ALL)) {
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(entry.getValue().getLookupId());
dataRec[i] = bd.getValue();
i++;
}
}
} else {
for (Long pfaf : map.keySet()) {
ArrayList<Long> vgbpfafs = template
.getVGBsInAggregate(pfaf,
record.getSiteKey(), key);
if (vgbpfafs != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(pfaf);
dataRec[i] = bd.getValue();
i++;
}
}
}
// NAME | GROUP | array |Dimension | dimensions
IDataRecord rec = new FloatDataRecord(key,
record.getDataURI() + "/" + domain.getCwa(),
dataRec, 1, new long[] { size });
dataStore.addDataRecord(rec);
}
float[] dataRec = new float[size];
int i = 0;
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(entry.getValue().getLookupId());
dataRec[i] = bd.getValue();
i++;
}
}
// NAME | GROUP | array |Dimension | dimensions
IDataRecord rec = new FloatDataRecord(FFMPRecord.ALL,
record.getDataURI() + "/" + domain.getCwa(),
dataRec, 1, new long[] { size });
dataStore.addDataRecord(rec);
} else {
statusHandler.handle(Priority.DEBUG, "No VGB's in domain: "
+ domain.getCwa());
@ -170,44 +144,40 @@ public class FFMPDao extends PluginDao {
else {
for (String huc : record.getBasinsMap().keySet()) {
if (record.getBasinData() != null) {
if (record.getBasinData(huc) != null) {
for (DomainXML domain : template.getDomains()) {
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(),
FFMPRecord.ALL);
FFMPBasinData fbd = record.getBasinData();
// ignore data outside domain
if (map.size() > 0 && fbd.getBasins().size() > 0) {
int size = map.size();
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(), huc);
FFMPBasinData fbd = record.getBasinData(huc);
// ignore data outside domain
if (map.size() > 0 && fbd.getBasins().size() > 0) {
int size = map.size();
float[] dataRec = new float[size];
int i = 0;
// write individual basins, use template, preserves
// ordering
for (Long pfaf : map.keySet()) {
FFMPBasin bd = fbd.get(pfaf);
if (bd != null) {
dataRec[i] = bd.getValue();
i++;
}
float[] dataRec = new float[size];
int i = 0;
// write individual basins, use template, preserves
// ordering
for (Long pfaf : map.keySet()) {
FFMPBasin bd = fbd.get(pfaf);
if (bd != null) {
dataRec[i] = bd.getValue();
i++;
}
// NAME | GROUP | array |Dimension | dimensions
if (i > 0) {
IDataRecord rec = new FloatDataRecord(huc,
record.getDataURI() + "/"
+ domain.getCwa(), dataRec, 1,
new long[] { size });
dataStore.addDataRecord(rec);
}
} else {
statusHandler.handle(
Priority.DEBUG,
"Data outside of domain: "
+ domain.getCwa());
}
// NAME | GROUP | array |Dimension | dimensions
if (i > 0) {
IDataRecord rec = new FloatDataRecord(
FFMPRecord.ALL, record.getDataURI() + "/"
+ domain.getCwa(), dataRec, 1,
new long[] { size });
dataStore.addDataRecord(rec);
}
} else {
statusHandler.handle(Priority.DEBUG,
"Data outside of domain: " + domain.getCwa());
}
}
}

6
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java
View file

@ -64,6 +64,7 @@ import com.vividsolutions.jts.geom.Geometry;
* ------------ ---------- ----------- --------------------------
* Jan 24, 2013 1552 mpduff Initial creation
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -126,8 +127,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory {
for (Map.Entry<String, Object> es : map.entrySet()) {
FFMPRecord rec = (FFMPRecord) es.getValue();
try {
rec.retrieveMapFromDataStore(templates, (String) request
.getIdentifiers().get(HUC));
rec.retrieveMapFromDataStore(templates);
} catch (Exception e) {
throw new DataRetrievalException(
"Failed to retrieve the IDataRecord for PluginDataObject: "
@ -204,7 +204,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory {
String siteKey = (String) request.getIdentifiers().get(SITE_KEY);
String cwa = (String) request.getIdentifiers().get(WFO);
FFMPBasinData basinData = rec.getBasinData(huc);
FFMPBasinData basinData = rec.getBasinData();
Map<Long, FFMPBasin> basinDataMap = basinData.getBasins();

521
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -32,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.regex.Pattern;
import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage;
import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage;
import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
import com.raytheon.edex.urifilter.URIFilter;
import com.raytheon.edex.urifilter.URIGenerateMessage;
@ -123,6 +123,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
* 02/25/13 1660 D. Hladky Redesigned data flow for FFTI in order to have only one mosaic piece in memory at a time.
* 03/13/13 1478 D. Hladky non-FFTI mosaic containers weren't getting ejected. Made it so that they are ejected after processing as well.
* 03/22/13 1803 D. Hladky Fixed broken performance logging for ffmp.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -131,9 +132,10 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
public class FFMPGenerator extends CompositeProductGenerator implements
MonitorConfigListener {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPGenerator.class);
/**
* Public constructor for FFMPGenerator
*
@ -152,13 +154,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private static final String productType = "ffmp";
/**
* The thought was this will eventually be dynamic when front end can support it.
* The thought was this will eventually be dynamic when front end can
* support it.
*/
public static final int SOURCE_CACHE_TIME = 6;
/**
* The thought was this will eventually be dynamic, static in AWIPS I.
* This is the time back limit for Flash Flood Guidance sources
* The thought was this will eventually be dynamic, static in AWIPS I. This
* is the time back limit for Flash Flood Guidance sources
*/
public static final int FFG_SOURCE_CACHE_TIME = 24;
@ -388,7 +391,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.PROBLEM,
"Couldn't create FFMP Filter.." + " primary Domain: "
+ domain.getCwa()
+ " this RUNNER is not a viable FFMP config.", e);
+ " this RUNNER is not a viable FFMP config.",
e);
}
}
@ -440,7 +444,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
@Override
public void generateProduct(URIGenerateMessage genMessage) {
if (loaded) {
if (loaded) {
try {
long time = System.currentTimeMillis();
this.config = new FFMPConfig(
@ -477,7 +481,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
} catch (InterruptedException e) {
statusHandler.handle(Priority.ERROR, "Process thread had been interupted!", e);
statusHandler.handle(Priority.ERROR,
"Process thread had been interupted!", e);
}
}
@ -554,6 +559,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private class ProcessProduct implements Runnable {
SourceXML ffmpProduct;
FFMPGenerator generator;
@Override
@ -592,7 +598,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
HashMap<String, Object> dataHash = config.getSourceData(ffmpProduct
.getSourceName());
FFMPRunXML runner = getRunConfig().getRunner(config.getCWA());
// process all of the dataKeys for this source
@ -684,19 +690,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements
ffmpRec, template);
ffmpRec = ffmp.processFFMP(ffmpProduct);
ffmpRec.constructDataURI();
if (ffmpRec != null) {
persistRecord(ffmpRec);
processDataContainer(ffmpRec, siteKey);
// Now that we have the data container,
// Now that we have the data container,
// we can process FFTI for this piece of the mosaic
if (ffmp.isFFTI()) {
fftiDone = false;
FFTISourceXML fftiSource = ffmp.getFFTISource();
// This only runs once for the site key loop
if (!fftiSources.contains(fftiSource)) {
FFTIProcessor ffti = new FFTIProcessor(
@ -735,16 +741,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
}
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
String sourceSiteDataKey = getSourceSiteDataKey(source,
dataKey, ffmpRec);
String sourceSiteDataKey = getSourceSiteDataKey(
source, dataKey, ffmpRec);
ffmpData.remove(sourceSiteDataKey);
statusHandler.info("Removing from memory: "+sourceSiteDataKey);
statusHandler.info("Removing from memory: "
+ sourceSiteDataKey);
}
}
}
@ -970,8 +977,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteSourceFileName(sourceId));
try {
sbl = SerializationUtil
.transformFromThrift(SourceBinList.class, FileUtil.file2bytes(f.getFile(), true));
sbl = SerializationUtil.transformFromThrift(SourceBinList.class,
FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName());
@ -979,8 +986,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR,
"Unable to read file " + f.getName());
} catch (IOException ioe) {
statusHandler.handle(Priority.ERROR, "General IO problem with file "
+ f.getName(), ioe);
statusHandler.handle(Priority.ERROR,
"General IO problem with file " + f.getName(), ioe);
}
return sbl;
@ -992,8 +999,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public String getAbsoluteSourceFileName(String sourceId) {
return productType + File.separator + "sources" + File.separator + sourceId
+ ".bin";
return productType + File.separator + "sources" + File.separator
+ sourceId + ".bin";
}
/**
@ -1098,7 +1105,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
*/
public FFMPDataContainer getFFMPDataContainer(String siteSourceKey,
ArrayList<String> hucs, Date backDate) {
Date backDate) {
FFMPDataContainer container = ffmpData.get(siteSourceKey);
@ -1112,7 +1119,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
siteKey = parts[0];
}
container = loadFFMPDataContainer(siteSourceKey, hucs, siteKey,
container = loadFFMPDataContainer(siteSourceKey, siteKey,
config.getCWA(), backDate);
if (container != null) {
@ -1194,180 +1201,157 @@ public class FFMPGenerator extends CompositeProductGenerator implements
/**
* Process the ffmp data container
*
* @param ffmpRec
* @param productKey
*/
public void processDataContainer(FFMPRecord ffmpRec, String productKey) {
public void processDataContainer(FFMPRecord ffmpRec, String productKey) {
String sourceName = null;
Date backDate = null;
String sourceSiteDataKey = null;
FFMPDataContainer fdc = null;
boolean write = true;
String sourceName = null;
Date backDate = null;
String sourceSiteDataKey = null;
FFMPDataContainer fdc = null;
boolean write = true;
try {
// write out the fast loader cache file
long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
String dataKey = ffmpRec.getDataKey();
try {
// write out the fast loader cache file
long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
String dataKey = ffmpRec.getDataKey();
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
// deal with setting of needed HUCS
ArrayList<String> hucs = template.getTemplateMgr().getHucLevels();
// pull from disk if there
fdc = getFFMPDataContainer(sourceSiteDataKey, backDate);
if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType())
|| source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
hucs.clear();
hucs.add(FFMPRecord.ALL);
} else {
hucs.remove(FFMPRecord.VIRTUAL);
}
// brand new or initial load up
if (fdc == null || !loadedData.contains(sourceSiteDataKey)) {
// pull from disk if there
fdc = getFFMPDataContainer(sourceSiteDataKey, hucs, backDate);
long time = System.currentTimeMillis();
fdc = new FFMPDataContainer(sourceSiteDataKey);
fdc = FFTIProcessor.populateDataContainer(fdc, template,
backDate, ffmpRec.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec.getSiteKey());
// brand new or initial load up
if (fdc == null || !loadedData.contains(sourceSiteDataKey)) {
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Populated new source: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
long time = System.currentTimeMillis();
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs);
fdc = FFTIProcessor.populateDataContainer(fdc, template, hucs,
backDate, ffmpRec.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec.getSiteKey());
} else {
if (source.getSourceType().equals(
SOURCE_TYPE.GAGE.getSourceType())
|| source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
hucs.clear();
hucs.add(FFMPRecord.ALL);
} else {
hucs.remove(FFMPRecord.VIRTUAL);
}
long time = System.currentTimeMillis();
// guidance sources are treated as a mosaic and are handled
// differently. They are force read at startup.
// This is the main line sequence a source will take when
// updated.
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Populated new source: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
Date newDate = fdc.getNewest();
Date oldDate = fdc.getOldest();
} else {
if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in
// existing source container, this will fill in
// gaps
// if
// they exist.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, newDate, ffmpRec.getDataTime()
.getRefTime(), ffmpRec.getWfo(),
source, ffmpRec.getSiteKey());
long time = System.currentTimeMillis();
// guidance sources are treated as a mosaic and are handled
// differently. They are force read at startup.
// This is the main line sequence a source will take when
// updated.
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
} else if (oldDate
.after(new Date(
backDate.getTime()
- (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for
// existing source container, this happens if
// the
// ingest was turned off for some period of
// time.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, backDate, oldDate,
ffmpRec.getWfo(), source,
ffmpRec.getSiteKey());
}
}
Date newDate = fdc.getNewest();
Date oldDate = fdc.getOldest();
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Checked Source files: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
}
}
if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in
// existing source container, this will fill in
// gaps
// if
// they exist.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, null, newDate, ffmpRec
.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec
.getSiteKey());
// add current record data
} else if (oldDate
.after(new Date(
backDate.getTime()
- (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for
// existing source container, this happens if
// the
// ingest was turned off for some period of
// time.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, null, backDate, oldDate,
ffmpRec.getWfo(), source,
ffmpRec.getSiteKey());
}
}
fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source,
ffmpRec.getBasinData(), ffmpRec.getSiteKey());
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Checked Source files: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
}
}
// cache it temporarily for FFTI use
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
// only write last one
write = false;
// add current record data
for (String huc : hucs) {
fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source,
ffmpRec.getBasinData(huc), huc, ffmpRec.getSiteKey());
}
if (!ffmpData.containsKey(sourceSiteDataKey)) {
ffmpData.put(sourceSiteDataKey, fdc);
} else {
ffmpData.replace(sourceSiteDataKey, fdc);
}
}
// cache it temporarily for FFTI use
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
// only write last one
write = false;
statusHandler.handle(
Priority.INFO,
"Processed FFMPDataContainer: in "
+ (System.currentTimeMillis() - ptime)
+ " ms: source: " + sourceSiteDataKey);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"Failed Processing FFMPDataContainer" + e.getMessage());
if (!ffmpData.containsKey(sourceSiteDataKey)) {
ffmpData.put(sourceSiteDataKey, fdc);
} else {
ffmpData.replace(sourceSiteDataKey, fdc);
}
}
} finally {
// purge it up
if (fdc != null) {
// this is defensive for if errors get thrown
if (backDate == null) {
backDate = new Date((System.currentTimeMillis())
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
statusHandler.handle(
Priority.INFO,
"Processed FFMPDataContainer: in "
+ (System.currentTimeMillis() - ptime)
+ " ms: source: " + sourceSiteDataKey);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"Failed Processing FFMPDataContainer" + e.getMessage());
if (!fdc.isPurged()) {
fdc.purge(backDate);
}
} finally {
// purge it up
if (fdc != null) {
// this is defensive for if errors get thrown
if (backDate == null) {
backDate = new Date((System.currentTimeMillis())
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
if (!fdc.isPurged()) {
fdc.purge(backDate);
}
if (write) {
// write it out
writeAggregateRecord(fdc, sourceSiteDataKey);
}
}
}
}
if (write) {
// write it out
writeAggregateRecord(fdc, sourceSiteDataKey);
}
}
}
}
/**
* load existing container
@ -1379,7 +1363,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public FFMPDataContainer loadFFMPDataContainer(String sourceSiteDataKey,
ArrayList<String> hucs, String siteKey, String wfo, Date backDate) {
String siteKey, String wfo, Date backDate) {
FFMPDataContainer fdc = null;
FFMPAggregateRecord record = null;
@ -1396,7 +1380,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// condition for first time read in
if (fdc == null && record != null) {
// creates a place holder for this source
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs, record);
fdc = new FFMPDataContainer(sourceSiteDataKey, record);
populated = true;
}
@ -1404,9 +1388,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (record != null && !populated) {
fdc.setAggregateData(record);
}
// sometimes a record will sit around for a long time and it will have data going back to the last precip event
// this can be an enormous amount of time. Want to get the data dumped from memory ASAP.
// sometimes a record will sit around for a long time and it will have
// data going back to the last precip event
// this can be an enormous amount of time. Want to get the data dumped
// from memory ASAP.
if (fdc != null) {
fdc.purge(backDate);
}
@ -1443,9 +1429,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
*
* @param fdc
*/
public void writeAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) {
public void writeAggregateRecord(FFMPDataContainer fdc,
String sourceSiteDataKey) {
WriteAggregateRecord writer = new WriteAggregateRecord(fdc, sourceSiteDataKey);
WriteAggregateRecord writer = new WriteAggregateRecord(fdc,
sourceSiteDataKey);
writer.run();
}
@ -1458,7 +1446,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private class WriteAggregateRecord implements Runnable {
private FFMPDataContainer fdc;
private String sourceSiteDataKey;
public void run() {
@ -1470,10 +1458,12 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
public WriteAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) {
public WriteAggregateRecord(FFMPDataContainer fdc,
String sourceSiteDataKey) {
this.fdc = fdc;
this.sourceSiteDataKey = sourceSiteDataKey;
statusHandler.handle(Priority.DEBUG, "Created Aggregate Record Writer");
statusHandler.handle(Priority.DEBUG,
"Created Aggregate Record Writer");
}
/**
@ -1492,34 +1482,37 @@ public class FFMPGenerator extends CompositeProductGenerator implements
aggRecord.setWfo(config.getCWA());
// times for Guidance basins will be null
aggRecord.setTimes(fdc.getOrderedTimes());
for (FFMPBasinData fbd : fdc.getBasinMap().values()) {
fbd.serialize();
aggRecord.addBasinData(fbd);
}
fdc.getBasinData().serialize();
aggRecord.setBasins(fdc.getBasinData());
}
if (aggRecord.getBasinsMap().size() > 0) {
if (aggRecord.getBasins() != null) {
try {
StorageProperties sp = null;
String compression = PluginRegistry.getInstance()
.getRegisteredObject(productType).getCompression();
.getRegisteredObject(productType)
.getCompression();
if (compression != null) {
sp = new StorageProperties();
sp.setCompression(Compression.valueOf(compression));
}
byte[] bytes = SerializationUtil.transformToThrift(aggRecord);
byte[] bytes = SerializationUtil
.transformToThrift(aggRecord);
// NAME | GROUP | array |Dimension | size
IDataRecord rec = new ByteDataRecord(sourceSiteDataKey, config.getCWA(),
bytes, 1, new long[] { bytes.length });
File hdf5File = FFMPUtils.getHdf5File(config.getCWA(), sourceSiteDataKey);
IDataStore dataStore = DataStoreFactory.getDataStore(hdf5File);
// write it, allowing, and in fact encouraging replacing the last one
IDataRecord rec = new ByteDataRecord(sourceSiteDataKey,
config.getCWA(), bytes, 1,
new long[] { bytes.length });
File hdf5File = FFMPUtils.getHdf5File(config.getCWA(),
sourceSiteDataKey);
IDataStore dataStore = DataStoreFactory
.getDataStore(hdf5File);
// write it, allowing, and in fact encouraging replacing
// the last one
dataStore.addDataRecord(rec, sp);
dataStore.store(StoreOp.OVERWRITE);
@ -1538,7 +1531,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
@Override
public synchronized void configChanged(MonitorConfigEvent fce) {
@ -1703,8 +1695,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public String getAbsoluteFFTIFileName(String fftiName) {
return productType + File.separator + "ffti" + File.separator + fftiName
+ ".bin";
return productType + File.separator + "ffti" + File.separator
+ fftiName + ".bin";
}
/**
@ -1734,7 +1726,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
public ConcurrentHashMap<String, FFTIData> getFFTIDataContainer() {
return fftiData;
}
/**
* Get value for an individual piece of the puzzle
*
@ -1748,8 +1740,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
public FFTIAccum getAccumulationForSite(String fftiSourceKey,
String fftiSiteKey, String fftiDataKey, double duration, String unit) {
SourceXML ffmpSource = getSourceConfig()
.getSourceByDisplayName(fftiSourceKey);
SourceXML ffmpSource = getSourceConfig().getSourceByDisplayName(
fftiSourceKey);
FFTIAccum accumulator = null;
String siteDataKey = ffmpSource.getDisplayName() + "-" + fftiSiteKey
+ "-" + fftiDataKey;
@ -1775,24 +1767,23 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long cur = config.getDate().getTime();
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
FFMPDataContainer fdc = null;
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
fdc = getFFMPDataContainer(siteDataKey, hucs, backDate);
fdc = getFFMPDataContainer(siteDataKey, backDate);
if (fdc != null) {
FFMPBasinData fbd = fdc.getBasinData(FFMPRecord.ALL);
FFMPBasinData fbd = fdc.getBasinData();
// go over the list of CWAs gathering the pfaf list
ArrayList<Long> pfafs = new ArrayList<Long>();
ArrayList<String> cwaList = config.fdm.getCwaList();
Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(), duration, fftiSiteKey);
Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(),
duration, fftiSiteKey);
if (!Double.isNaN(gap)) {
for (Long key : fbd.getBasins().keySet()) {
@ -1831,14 +1822,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
ffmpData.remove(siteDataKey);
statusHandler.info("Removing from memory: "+siteDataKey);
statusHandler.info("Removing from memory: " + siteDataKey);
accumulator.setReset(false);
writeFFTIData(siteDataKey, accumulator);
}
return accumulator;
}
/**
* Gets the ratio and difference values for this site
*
@ -1880,24 +1871,24 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long cur = config.getDate().getTime();
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
// make sure we have data
Date ffgBackDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
Date ffgBackDate = new Date(
config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
String primarySource = fscm.getPrimarySource(ffmpQSource);
ProductXML product = fscm.getProduct(primarySource);
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
FFMPDataContainer guidContainer = getFFMPDataContainer(
ffgType, hucs, ffgBackDate);
FFMPDataContainer guidContainer = getFFMPDataContainer(ffgType,
ffgBackDate);
long guidSourceExpiration = 0l;
if (guidContainer == null) {
guidContainer = new FFMPDataContainer(ffgType, hucs);
guidContainer = new FFMPDataContainer(ffgType);
}
for (SourceXML iguidSource : product
@ -1905,7 +1896,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (guidSourceExpiration == 0l) {
guidSourceExpiration = iguidSource
.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
.getExpirationMinutes(qSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
break;
}
}
@ -1918,15 +1910,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements
+ " " + qSourceKey + " " + " comparison.");
return values;
}
String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-" + qSiteKey + "-"+ qSiteKey;
FFMPDataContainer qpeContainer = getFFMPDataContainer(qpeSiteSourceDataKey, hucs, backDate);
String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-"
+ qSiteKey + "-" + qSiteKey;
FFMPDataContainer qpeContainer = getFFMPDataContainer(
qpeSiteSourceDataKey, backDate);
if (qpeContainer != null) {
// go over the list of CWAs gathering the pfaf list
ArrayList<Long> pfafs = new ArrayList<Long>();
ArrayList<String> cwaList = config.fdm.getCwaList();
FFMPBasinData fbd = qpeContainer.getBasinData(FFMPRecord.ALL);
FFMPBasinData fbd = qpeContainer.getBasinData();
for (Long key : fbd.getBasins().keySet()) {
for (String cwa : cwaList) {
@ -1953,22 +1947,22 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
Double gap = FFTI.getGap(qpeContainer, ffmpQSource, config.getDate(), duration,
qSiteKey);
Double gap = FFTI.getGap(qpeContainer, ffmpQSource,
config.getDate(), duration, qSiteKey);
if (!Double.isNaN(gap)) {
List<Float> qpes = qpeContainer.getBasinData(FFMPRecord.ALL)
List<Float> qpes = qpeContainer.getBasinData()
.getAccumValues(pfafs, backDate, config.getDate(),
expirationTime, false);
FFMPGuidanceInterpolation interpolator = new FFMPGuidanceInterpolation(
fscm, product, frcm.getRunner(
config.getCWA()).getProduct(qSiteKey),
primarySource, ffgType, qSiteKey);
fscm, product, frcm.getRunner(config.getCWA())
.getProduct(qSiteKey), primarySource,
ffgType, qSiteKey);
interpolator.setInterpolationSources(duration);
List<Float> guids = guidContainer.getBasinData(FFMPRecord.ALL)
List<Float> guids = guidContainer.getBasinData()
.getGuidanceValues(pfafs, interpolator,
guidSourceExpiration);
@ -1982,44 +1976,41 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// replace or insert it
ffmpData.remove(qpeSiteSourceDataKey);
statusHandler.info("Removing from memory: "+qpeSiteSourceDataKey);
statusHandler.info("Removing from memory: " + qpeSiteSourceDataKey);
values.setReset(false);
writeFFTIData(siteDataKey, values);
}
return values;
}
/**
* Persist the record that has finished processing.
* This is different than other DAT tools.
* Other tools wait until all are finished processing
* before persisting. FFMP persists as it goes in order
* to lessen the data surge being sent to pypies.
* Persist the record that has finished processing. This is different than
* other DAT tools. Other tools wait until all are finished processing
* before persisting. FFMP persists as it goes in order to lessen the data
* surge being sent to pypies.
*
* @param record
* @return
*/
private synchronized void persistRecord(FFMPRecord record) {
// persist out this record
try {
setPluginDataObjects(new FFMPRecord[]{record});
setPluginDao(new FFMPDao(getCompositeProductType(),
template, fscm, config.getCWA()));
setPluginDataObjects(new FFMPRecord[] { record });
setPluginDao(new FFMPDao(getCompositeProductType(), template, fscm,
config.getCWA()));
persistRecords();
fireTopicUpdate();
// clear out pdos that are written
pdos = null;
} catch (PluginException e) {
statusHandler.handle(Priority.PROBLEM, "Couldn't persist the record.", e);
statusHandler.handle(Priority.PROBLEM,
"Couldn't persist the record.", e);
}
}
/**
* Find siteSourceDataKey
*
@ -2028,25 +2019,25 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @param ffmpRec
* @return
*/
private String getSourceSiteDataKey(SourceXML source, String dataKey, FFMPRecord ffmpRec) {
private String getSourceSiteDataKey(SourceXML source, String dataKey,
FFMPRecord ffmpRec) {
String sourceName = source.getSourceName();
String sourceSiteDataKey = null;
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
if (source.getSourceType().equals(SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() + "-"
+ dataKey;
}
return sourceSiteDataKey;
}
/**
* Log process statistics
*
@ -2083,4 +2074,4 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
}
}

21
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java
View file

@ -20,7 +20,6 @@ package com.raytheon.uf.edex.plugin.ffmp.common;
* further licensing information.
**/
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
@ -45,6 +44,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* 29 July, 2012 578 dhladky memory work
* 27 Jan, 2013 1478 dhladky Changed arraylist to list for times, more constants
* 02/01/13 1569 D. Hladky Added constants
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -93,15 +93,14 @@ public class FFMPInterpolatedGuidanceDelay {
public boolean calculateDelayedGuidance() {
boolean delayGuidance = false;
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
FFMPDataContainer qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName()
+ "-" + siteKey + "-" + siteKey, hucs, backDate);
FFMPDataContainer qpeContainer = generator.getFFMPDataContainer(
qpeSource.getSourceName() + "-" + siteKey + "-" + siteKey,
backDate);
// Don't do anything, we have no QPE
if (qpeContainer != null) {
long expirationTime = qpeSource.getExpirationMinutes(siteKey)
* TimeUtil.MILLIS_PER_MINUTE;
// determine lag_time
@ -115,8 +114,8 @@ public class FFMPInterpolatedGuidanceDelay {
.getOrderedTimes(currentRecord.getDataTime().getRefTime());
// EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld)
for (Entry<Long, FFMPBasin> entry : currentRecord.getBasinsMap()
.get(FFMPRecord.ALL).getBasins().entrySet()) {
for (Entry<Long, FFMPBasin> entry : currentRecord.getBasinData()
.getBasins().entrySet()) {
FFMPBasin currBasin = entry.getValue();
FFMPGuidanceBasin oldBasin = (FFMPGuidanceBasin) previousGuidanceData
.get(entry.getKey());
@ -136,8 +135,8 @@ public class FFMPInterpolatedGuidanceDelay {
// this is essentially a ratio of the first accumulation
// step increment
// to the total amount over this time window.
FFMPBasin qpeBasin = qpeContainer.getBasinData(
FFMPRecord.ALL).get(entry.getKey());
FFMPBasin qpeBasin = qpeContainer.getBasinData().get(
entry.getKey());
if (qpeBasin != null) {
float intervalAccum = qpeBasin.getAccumValue(
@ -160,7 +159,7 @@ public class FFMPInterpolatedGuidanceDelay {
currBasin.setValue(newDate, val);
}
}
delayGuidance = true;
}

107
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java
View file

@ -98,6 +98,7 @@ import com.vividsolutions.jts.geom.Polygon;
* 02/01/13 1569 D. Hladky Added constants
* 02/25/13 1660 D. Hladky FFTI design change to help mosaic processing.
* 05/01/2013 15684 zhao Unlock when Exception caught
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -214,29 +215,6 @@ public class FFMPProcessor {
processSource();
}
// don't do gages for anything other than "ALL"
if ((sourceId != null)
&& !source.getSourceType().equals(
FFMPSourceConfigurationManager.SOURCE_TYPE.GAGE
.getSourceType())) {
//ArrayList<String> hucs = template.getTemplateMgr().getHucLevels();// DR 15514
String[] hucs = template.getTemplateMgr().getHucLevelsInArray();// DR 15514
synchronized (hucs) {
if (hucs != null) {
//for (String huc : hucs) {
for(int i=0; i<hucs.length; i++){
String huc = hucs[i];
if (huc != null) {
if (!huc.equals("ALL") || !huc.equals("VIRTUAL")) {
setValues(huc);
}
}
}
}
}
}
statusHandler.handle(Priority.INFO,
"Processed Source: " + ffmpRec.getSourceName() + " sitekey: "
+ siteKey + " dataKey: " + dataKey + " time: "
@ -471,7 +449,7 @@ public class FFMPProcessor {
if (sourceId != null) {
for (Long key : map.keySet()) {
FFMPBasin basin = getBasin(key, FFMPRecord.ALL);
FFMPBasin basin = getBasin(key);
Date date = null;
Float val = null;
@ -595,7 +573,7 @@ public class FFMPProcessor {
for (Long key : map.keySet()) {
FFMPBasin basin = getBasin(key, "ALL");
FFMPBasin basin = getBasin(key);
float val = 0.0f;
val = processGrib(key, domain.getCwa());
setBasin(basin, val);
@ -643,11 +621,9 @@ public class FFMPProcessor {
Date backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()-(FFMPGenerator.SOURCE_CACHE_TIME * TimeUtil.MILLIS_PER_HOUR));
ArrayList<String> hucs = new ArrayList<String>();
hucs.add("ALL");
FFMPDataContainer ffgContainer = generator
.getFFMPDataContainer(sourceNameString, hucs,
.getFFMPDataContainer(sourceNameString,
backDate);
if (ffgContainer != null
@ -688,7 +664,7 @@ public class FFMPProcessor {
siteKey, guidFrequency, source,
qpeSource, previousDate, recdate,
generator,
ffgContainer.getBasinData(FFMPRecord.ALL),
ffgContainer.getBasinData(),
ffmpRec);
boolean delayGuidance = figd
@ -811,9 +787,9 @@ public class FFMPProcessor {
*
* @param ffmp
*/
private FFMPBasinData getBasinData(String huc) {
private FFMPBasinData getBasinData() {
return ffmpRec.getBasinData(huc);
return ffmpRec.getBasinData();
}
/**
@ -823,15 +799,11 @@ public class FFMPProcessor {
* @param huc
* @return
*/
private FFMPBasin getBasin(Long pfaf, String huc) {
FFMPBasin basin = getBasinData(huc).get(pfaf);
private FFMPBasin getBasin(Long pfaf) {
FFMPBasin basin = getBasinData().get(pfaf);
if (basin == null) {
if (huc.equals(FFMPRecord.ALL)) {
basin = new FFMPBasin(pfaf, false);
} else {
basin = new FFMPBasin(pfaf, true);
}
getBasinData(huc).put(pfaf, basin);
basin = new FFMPBasin(pfaf, false);
getBasinData().put(pfaf, basin);
}
return basin;
}
@ -845,64 +817,15 @@ public class FFMPProcessor {
*/
private FFMPVirtualGageBasin getVirtualBasin(String lid, Long pfaf,
String huc) {
FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData(huc)
FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData()
.get(pfaf);
if (basin == null) {
if (huc.equals(FFMPRecord.ALL)) {
basin = new FFMPVirtualGageBasin(lid, pfaf, false);
} else {
basin = new FFMPVirtualGageBasin(lid, pfaf, true);
}
getBasinData(huc).put(pfaf, basin);
basin = new FFMPVirtualGageBasin(lid, pfaf, false);
getBasinData().put(pfaf, basin);
}
return basin;
}
/**
* Sets the values for the aggregated basins
*
* @param type
* @return
*/
private void setValues(String huc) {
try {
// Get basins for level, we process VGB's differently because it is
// a
// special case
if (!huc.equals(FFMPRecord.VIRTUAL) && !huc.equals(FFMPRecord.ALL)) {
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(siteKey,
domain.getCwa(), huc);
for (Long pfaf : map.keySet()) {
FFMPBasin basin = getBasin(pfaf, huc);
Float val = 0.0f;
// average values
try {
ArrayList<Long> aggPfafs = template
.getAggregatePfafs(pfaf, siteKey, huc);
ArrayList<Double> areas = template
.getAreas(aggPfafs);
val = ffmpRec.getBasinData(FFMPRecord.ALL).getAverageValue(
aggPfafs, areas);
} catch (Exception e) {
// Value is NAN, ignore it.
}
basin.setValue(config.getDate(), val);
}
}
}
} catch (Exception e) {
config.getGenerator().logger.error("Unable to process " + huc
+ " level data");
}
}
/**
* Process the PDO for a geometry
*

30
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java
View file

@ -57,6 +57,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* </pre>
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Apr 18, 2013 1919 dhladky Fixed VGB breakage
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* @author dhladky
* @version 1.0
@ -182,8 +183,8 @@ public class FFTIProcessor {
*/
public static FFMPDataContainer populateDataContainer(
FFMPDataContainer sourceContainer, FFMPTemplates template,
ArrayList<String> hucs, Date startDate, Date endDate, String wfo,
SourceXML source, String siteKey) {
Date startDate, Date endDate, String wfo, SourceXML source,
String siteKey) {
ArrayList<String> uris = getUris(startDate, endDate, wfo, source,
siteKey);
@ -209,19 +210,11 @@ public class FFTIProcessor {
if (!contains) {
try {
if (hucs == null) {
hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
}
for (String huc : hucs) {
rec = populateRecord(rec, huc, template);
FFMPBasinData newData = rec.getBasinData(huc);
sourceContainer.addFFMPEntry(rec.getDataTime()
.getRefTime(), source, newData, huc, siteKey);
}
rec = populateRecord(rec, template);
FFMPBasinData newData = rec.getBasinData();
sourceContainer.addFFMPEntry(
rec.getDataTime().getRefTime(), source, newData,
siteKey);
// System.out.println("Adding Time: "
// + rec.getDataTime().getRefTime());
@ -309,8 +302,7 @@ public class FFTIProcessor {
* @return
* @throws PluginException
*/
public static FFMPRecord populateRecord(FFMPRecord rec, String huc,
FFMPTemplates template) throws PluginException {
public static FFMPRecord populateRecord(FFMPRecord rec, FFMPTemplates template) throws PluginException {
try {
@ -328,9 +320,9 @@ public class FFTIProcessor {
// check for gage(VGB) types, if so process as a VGB
if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType())) {
rec.retrieveVirtualMapFromDataStore(template, huc);
rec.retrieveVirtualMapFromDataStore(template);
} else {
rec.retrieveMapFromDataStore(template, huc);
rec.retrieveMapFromDataStore(template);
}
// System.out.println("Size of huc: "