xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2184 Removed all other HUCs other than ALL from storage

Browse source 
Change-Id: I04885f5162cbe25b4345fe797da5a6f7ac246cc5

Former-commit-id: 251cf3cca7c6653300f115083aa860444fee0ea9
This commit is contained in:
Dave Hladky 2013-07-17 13:27:06 -05:00
parent 7a31fbb386
commit 84dae0a6b8
20 changed files with 1087 additions and 1178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java
View file

@ -97,6 +97,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen No longer starts loading threads, resourceData does that
* Jun 07, 2013 2075 njensen Extracted FFMPProcessUris to separate class
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -291,11 +292,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @param phuc
* @return
*/
protected List<String> getLoadedUris(String siteKey, String source,
String phuc) {
protected List<String> getLoadedUris(String siteKey, String source) {
FFMPSiteData siteData = siteDataMap.get(siteKey);
FFMPSourceData sourceData = siteData.getSourceData(source);
return sourceData.getLoadedUris(phuc);
return sourceData.getLoadedUris();
}
/**
@ -308,7 +308,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @throws VizException
*/
public void populateFFMPRecord(String siteKey, String dataKey,
String source, Date ptime, String phuc, boolean retrieveNew) {
String source, Date ptime, boolean retrieveNew) {
if (source != null) {
boolean dupOverride = false;
@ -325,11 +325,11 @@ public class FFMPMonitor extends ResourceMonitor {
for (String uri : uris) {
if (uri != null) {
if (dupOverride
|| !getLoadedUris(siteKey, source, phuc)
|| !getLoadedUris(siteKey, source)
.contains(uri)) {
try {
populateFFMPRecord(siteKey, new FFMPRecord(
uri), source, phuc);
uri), source);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, "
@ -353,10 +353,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @throws VizException
*/
public FFMPRecord populateFFMPRecord(String uri, String siteKey,
String source, String phuc) throws Exception {
String source) throws Exception {
try {
populateFFMPRecord(siteKey, new FFMPRecord(uri), source, phuc);
populateFFMPRecord(siteKey, new FFMPRecord(uri), source);
} catch (VizException e) {
statusHandler.handle(Priority.INFO,
"FFMP Can't retrieve FFMP URI, " + uri, e);
@ -387,31 +387,26 @@ public class FFMPMonitor extends ResourceMonitor {
FFMPRecord curRecord = sourceData.getRecord();
if (curRecord == null) {
// add each huc requested
for (String huc : data.getBasinsMap().keySet()) {
// add all of the uris
for (Entry<Date, List<String>> duris : uris.entrySet()) {
if (data.getTimes().contains(duris.getKey().getTime())) {
for (String uri : duris.getValue()) {
if (curRecord == null) {
curRecord = new FFMPRecord(uri);
sourceData.setRecord(curRecord);
}
sourceData.addLoadedUri(huc, uri);
// add all of the uris
for (Entry<Date, List<String>> duris : uris.entrySet()) {
if (data.getTimes().contains(duris.getKey().getTime())) {
for (String uri : duris.getValue()) {
if (curRecord == null) {
curRecord = new FFMPRecord(uri);
sourceData.setRecord(curRecord);
}
sourceData.addLoadedUri(uri);
}
}
}
}
if (curRecord != null) {
for (Entry<String, FFMPBasinData> entry : data.getBasinsMap()
.entrySet()) {
FFMPBasinData basinData = entry.getValue();
basinData.populate(data.getTimes());
curRecord.populate(basinData, entry.getKey());
}
FFMPBasinData basinData = data.getBasins();
basinData.populate(data.getTimes());
curRecord.populate(basinData);
}
}
@ -422,13 +417,13 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey
* @param ffmpRec
* @param source
* @param phuc
*
* @throws Exception
*/
public void populateFFMPRecord(String siteKey, FFMPRecord ffmpRec,
String source, String phuc) throws Exception {
String source) throws Exception {
FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source, phuc);
FFMPLoadRecord flr = new FFMPLoadRecord(siteKey, ffmpRec, source);
flr.run();
}
@ -446,12 +441,12 @@ public class FFMPMonitor extends ResourceMonitor {
String source, String phuc, FFMPBasin basin) throws VizException {
if (dataUri != null) {
List<String> uris = getLoadedUris(siteKey, source, phuc);
List<String> uris = getLoadedUris(siteKey, source);
if (!uris.contains(dataUri)) {
try {
SourceXML sourceXML = fscm.getSource(source);
FFMPRecord ffmpRec = populateFFMPRecord(dataUri, siteKey,
source, phuc);
source);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory.getDataStore(loc);
@ -698,14 +693,12 @@ public class FFMPMonitor extends ResourceMonitor {
*
*/
public FFMPRecord getFFMPData(ProductXML product, String siteKey,
String dataKey, String sourceName, Date ptime, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date ptime, boolean retrieveNew) {
FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName)
.getRecord();
if ((record != null)
&& (record.getBasinData(phuc).getBasins().size() > 0)) {
if ((record != null) && (record.getBasinData().getBasins().size() > 0)) {
SourceXML sourceXML = getSourceConfig().getSource(sourceName);
@ -731,8 +724,7 @@ public class FFMPMonitor extends ResourceMonitor {
continue;
} else {
populateFFMPRecord(siteKey, dataKey,
source.getSourceName(), ptime, phuc,
retrieveNew);
source.getSourceName(), ptime, retrieveNew);
}
}
} else {
@ -740,7 +732,7 @@ public class FFMPMonitor extends ResourceMonitor {
if (!siteDataMap.get(siteKey).getSourceData(sourceName)
.hasLoadedAnyUris()) {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
}
@ -748,7 +740,7 @@ public class FFMPMonitor extends ResourceMonitor {
.getRecord();
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime, phuc,
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
retrieveNew);
}
@ -775,17 +767,17 @@ public class FFMPMonitor extends ResourceMonitor {
.getGuidanceSources(product, guidSrc)) {
populateFFMPRecord(siteKey, dataKey,
ffgSource.getSourceName(), ptime, phuc,
ffgSource.getSourceName(), ptime,
retrieveNew);
}
} else {
populateFFMPRecord(siteKey, dataKey, sourceName,
ptime, phuc, retrieveNew);
ptime, retrieveNew);
}
}
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
} else {
// special case where FFG is the primary source
@ -799,7 +791,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = sourcexml.getDisplayName();
} else {
populateFFMPRecord(siteKey, dataKey, sourceName, ptime,
phuc, retrieveNew);
retrieveNew);
}
}
@ -1054,20 +1046,18 @@ public class FFMPMonitor extends ResourceMonitor {
* Get the pertinent QPE source Record.
*
* @param date
* @param phuc
* @param retrieveNew
* @return
*/
public FFMPRecord getQPERecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date date, boolean retrieveNew) {
// comparisons done with table display
if (product != null) {
sourceName = product.getQpe();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1095,12 +1085,11 @@ public class FFMPMonitor extends ResourceMonitor {
* Get the rate record.
*
* @param date
* @param phuc
* @param retrieveNew
* @return
*/
public FFMPRecord getRateRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
String dataKey, String sourceName, Date date,
boolean retrieveNew) {
// comparisons done with table display
@ -1108,7 +1097,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = product.getRate();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1142,8 +1131,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public FFMPRecord getQPFRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
boolean retrieveNew) {
String dataKey, String sourceName, Date date, boolean retrieveNew) {
FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance()
.getTableConfigData(siteKey);
@ -1157,7 +1145,7 @@ public class FFMPMonitor extends ResourceMonitor {
.getSourceName();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1210,8 +1198,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = source.getDisplayName();
}
return getFFMPData(product, siteKey, null, sourceName, date, phuc,
false);
return getFFMPData(product, siteKey, null, sourceName, date, false);
}
/**
@ -1226,7 +1213,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public Map<String, FFMPRecord> getGuidanceRecords(ProductXML product,
String siteKey, Date date, String phuc, boolean retrieveNew) {
String siteKey, Date date, boolean retrieveNew) {
Map<String, FFMPRecord> guidRecs = new HashMap<String, FFMPRecord>();
ProductRunXML productRun = FFMPRunConfigurationManager.getInstance()
@ -1236,7 +1223,7 @@ public class FFMPMonitor extends ResourceMonitor {
for (String type : guidTypes) {
FFMPRecord guidRec = getFFMPData(product, siteKey, null, type,
date, phuc, retrieveNew);
date, retrieveNew);
guidRecs.put(type, guidRec);
}
@ -1275,7 +1262,7 @@ public class FFMPMonitor extends ResourceMonitor {
* @return
*/
public FFMPRecord getVirtualRecord(ProductXML product, String siteKey,
String dataKey, String sourceName, Date date, String phuc,
String dataKey, String sourceName, Date date,
boolean retrieveNew) {
// comparisons done with table display
// field doesn't matter here
@ -1284,7 +1271,7 @@ public class FFMPMonitor extends ResourceMonitor {
sourceName = product.getVirtual();
}
return getFFMPData(product, siteKey, dataKey, sourceName, date, phuc,
return getFFMPData(product, siteKey, dataKey, sourceName, date,
retrieveNew);
}
@ -1305,7 +1292,7 @@ public class FFMPMonitor extends ResourceMonitor {
FFMPRecord record = siteDataMap.get(siteKey).getSourceData(sourceName)
.getRecord();
if (record != null) {
FFMPBasinData basinData = record.getBasinData(phuc);
FFMPBasinData basinData = record.getBasinData();
if (basinData != null) {
basin = basinData.get(pfaf);
}
@ -1350,7 +1337,7 @@ public class FFMPMonitor extends ResourceMonitor {
.get(siteKey)
.getSourceData(
source.getSourceName())
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey,
source.getSourceName(), phuc,
@ -1376,7 +1363,7 @@ public class FFMPMonitor extends ResourceMonitor {
for (String uri : uris) {
if (!siteDataMap.get(siteKey)
.getSourceData(sourceName)
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey, sourceName,
phuc, fgb);
@ -1397,7 +1384,7 @@ public class FFMPMonitor extends ResourceMonitor {
POINT_RETRIVAL: for (List<String> uris : availableUris.values()) {
for (String uri : uris) {
if (!siteDataMap.get(siteKey).getSourceData(sourceName)
.getLoadedUris(phuc).contains(uri)) {
.getLoadedUris().contains(uri)) {
// populate point only
populateFFMPBasin(uri, siteKey, sourceName, phuc,
basin);
@ -1593,21 +1580,15 @@ public class FFMPMonitor extends ResourceMonitor {
.hasLoadedAnyUris()) {
FFMPSourceData sourceData = siteData
.getSourceData(sourceName);
Set<String> hucs = sourceData
.getLoadedHucs();
for (String huc : hucs) {
sourceData.getLoadedUris(huc).remove(
uri);
}
sourceData.removeLoadedUri(uri);
}
}
} else {
FFMPSourceData sourceData = siteData
.getSourceData(fsource);
Set<String> hucs = sourceData.getLoadedHucs();
for (String huc : hucs) {
sourceData.getLoadedUris(huc).remove(uri);
}
sourceData.removeLoadedUri(uri);
}
}
}
@ -1625,16 +1606,14 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey
* @param sourceName
* @param barrierTime
* @param phuc
*/
public void processUri(String uri, String siteKey, String sourceName,
Date barrierTime, String phuc) {
Date barrierTime) {
if (uri != null) {
try {
FFMPRecord record = populateFFMPRecord(uri, siteKey,
sourceName, phuc);
FFMPRecord record = populateFFMPRecord(uri, siteKey, sourceName);
if (record != null) {
record.getBasinData(phuc).loadNow();
record.getBasinData().loadNow();
SourceXML source = getSourceConfig().getSource(sourceName);
if (source != null) {
record.setExpiration(source
@ -1656,10 +1635,10 @@ public class FFMPMonitor extends ResourceMonitor {
* @param sourceName
*/
public void processUris(NavigableMap<Date, List<String>> uriMap,
String siteKey, String sourceName, Date barrierTime, String phuc,
String siteKey, String sourceName, Date barrierTime,
SubMonitor smonitor) {
FFMPProcessUris processor = new FFMPProcessUris(this, uriMap, siteKey,
sourceName, barrierTime, phuc);
sourceName, barrierTime);
processor.run(smonitor);
}
@ -2126,16 +2105,13 @@ public class FFMPMonitor extends ResourceMonitor {
final String fsource;
final String fhuc;
final String fsiteKey;
public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec,
String source, String huc) throws Exception {
public FFMPLoadRecord(String siteKey, FFMPRecord ffmpRec, String source)
throws Exception {
this.fffmpRec = ffmpRec;
this.fsource = source;
this.fsiteKey = siteKey;
this.fhuc = huc;
}
public void run() {
@ -2143,7 +2119,7 @@ public class FFMPMonitor extends ResourceMonitor {
load();
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM, "FFMP load FFMPData, "
+ fsource + " " + fhuc, e);
+ fsource, e);
}
}
@ -2151,7 +2127,7 @@ public class FFMPMonitor extends ResourceMonitor {
if (fffmpRec != null) {
List<String> uris = getLoadedUris(fsiteKey, fsource, fhuc);
List<String> uris = getLoadedUris(fsiteKey, fsource);
String dataUri = fffmpRec.getDataURI();
if (!uris.contains(dataUri)) {
Date refTime = fffmpRec.getDataTime().getRefTime();
@ -2188,20 +2164,20 @@ public class FFMPMonitor extends ResourceMonitor {
try {
if (isGageSource && fhuc.equals(FFMPRecord.ALL)) {
if (isGageSource) {
curRecord.retrieveVirtualMapFromDataStore(loc,
dataUri, getTemplates(fsiteKey), refTime,
fffmpRec.getSourceName());
} else {
curRecord.retrieveMapFromDataStore(loc, dataUri,
getTemplates(fffmpRec.getSiteKey()), fhuc,
getTemplates(fffmpRec.getSiteKey()),
refTime, fffmpRec.getSourceName());
}
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, " + dataUri, e);
}
sourceData.addLoadedUri(fhuc, dataUri);
sourceData.addLoadedUri(dataUri);
}
}
}

12
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPProcessUris.java
View file

@ -44,6 +44,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 7, 2013 njensen Initial creation
* Jul 15, 2013 2184 dhladky Removed all HUC's but ALL
*
* </pre>
*
@ -64,18 +65,15 @@ public class FFMPProcessUris {
private final Date fbarrierTime;
private final String fhuc;
private final FFMPMonitor ffmpMonitor;
public FFMPProcessUris(FFMPMonitor ffmpMonitor,
NavigableMap<Date, List<String>> uriMap, String siteKey,
String sourceName, Date barrierTime, String phuc) {
String sourceName, Date barrierTime) {
this.furiMap = uriMap;
this.fsiteKey = siteKey;
this.fbarrierTime = barrierTime;
this.fsourceName = sourceName;
this.fhuc = phuc;
this.ffmpMonitor = ffmpMonitor;
}
@ -90,7 +88,7 @@ public class FFMPProcessUris {
isGuidance = true;
}
List<String> loadedUris = ffmpMonitor.getLoadedUris(fsiteKey,
fsourceName, fhuc);
fsourceName);
Set<FFMPRecord> populatedRecords = new HashSet<FFMPRecord>();
for (List<String> uris : furiMap.descendingMap().values()) {
for (String uri : uris) {
@ -102,7 +100,7 @@ public class FFMPProcessUris {
|| isGuidance) {
try {
record = ffmpMonitor.populateFFMPRecord(uri,
fsiteKey, fsourceName, fhuc);
fsiteKey, fsourceName);
if (record != null) {
populatedRecords.add(record);
if (source != null) {
@ -121,7 +119,7 @@ public class FFMPProcessUris {
monitor.beginTask(null, populatedRecords.size());
for (FFMPRecord record : populatedRecords) {
record.getBasinData(fhuc).loadNow();
record.getBasinData().loadNow();
monitor.worked(1);
}
}

67
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPSourceData.java
View file

@ -20,11 +20,9 @@
package com.raytheon.uf.viz.monitor.ffmp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentNavigableMap;
import java.util.concurrent.ConcurrentSkipListMap;
@ -42,6 +40,7 @@ import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
* ------------ ---------- ----------- --------------------------
* Feb 18, 2013 njensen Initial creation
* Feb 28, 2013 1729 dhladky Sped up, synch blocks were hanging it.
* Jul 15, 2013 2184 dhladky Removed all HUC's but ALL
*
* </pre>
*
@ -59,7 +58,7 @@ public class FFMPSourceData {
private ConcurrentNavigableMap<Date, List<String>> availableUris = new ConcurrentSkipListMap<Date, List<String>>();
/** map of huc to list of loaded URIs **/
private ConcurrentMap<String, List<String>> loadedUris = new ConcurrentHashMap<String, List<String>>();
private List<String> loadedUris = new ArrayList<String>();
/**
* Clears the data
@ -68,7 +67,9 @@ public class FFMPSourceData {
ffmpData = null;
previousUriQueryDate = null;
availableUris.clear();
loadedUris.clear();
synchronized (loadedUris) {
loadedUris.clear();
}
}
/**
@ -110,45 +111,23 @@ public class FFMPSourceData {
}
/**
* Gets the URIs associated with a HUC that have been loaded.
* Gets the URIs that have been loaded.
*
* @param huc
* @return
*/
public List<String> getLoadedUris(String huc) {
List<String> loaded = loadedUris.get(huc);
if (loaded == null) {
loaded = new ArrayList<String>();
List<String> previous = loadedUris.putIfAbsent(huc, loaded);
if (previous != null) {
return previous;
}
}
return loaded;
public List<String> getLoadedUris() {
return Collections.unmodifiableList(loadedUris);
}
/**
* Tracks a URI associated with a HUC as loaded.
* Tracks a URI as loaded.
*
* @param huc
* @param uri
*/
public void addLoadedUri(String huc, String uri) {
List<String> uriList = loadedUris.get(huc);
if (uriList == null) {
uriList = new ArrayList<String>();
List<String> previous = loadedUris.putIfAbsent(huc, uriList);
if (previous != null) {
uriList = previous;
}
public void addLoadedUri(String uri) {
synchronized (loadedUris) {
loadedUris.add(uri);
}
uriList.add(uri);
}
/**
@ -160,15 +139,6 @@ public class FFMPSourceData {
return !loadedUris.isEmpty();
}
/**
* Gets the set of HUCs that have loaded some URIs.
*
* @return
*/
public Set<String> getLoadedHucs() {
return loadedUris.keySet();
}
/**
* Gets the Available URIs based on time.
*
@ -178,4 +148,15 @@ public class FFMPSourceData {
return availableUris;
}
/**
* Removes a URI
*
* @param uri
*/
public void removeLoadedUri(String uri) {
synchronized (loadedUris) {
loadedUris.remove(uri);
}
}
}

245
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java
View file

@ -77,6 +77,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData;
* May 22, 2013 1902 mpduff Code cleanup.
* Jun 11, 2013 2085 njensen Extracted row creation to FFMPRowGenerator and
* multi-threaded row creation.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -164,134 +165,139 @@ public class FFMPDataGenerator {
public FFMPTableData generateFFMPData() throws Exception {
// You should always have at least a QPE data source
FFMPTableData tData = null;
// update the FFFGDataManager
FFFGDataMgr.getUpdatedInstance();
try {
FIELDS field = getBaseField();
if (field == null || baseRec == null) {
return tData;
}
FFMPBasinData fbd = null;
if (centeredAggregationKey != null) {
fbd = baseRec.getBasinData(FFMPRecord.ALL);
} else {
fbd = baseRec.getBasinData(huc);
}
tData = new FFMPTableData(fbd.getBasins().size());
List<DomainXML> domains = resource.getDomains();
if (!fbd.getBasins().isEmpty()) {
if ((centeredAggregationKey == null)
|| huc.equals(FFMPRecord.ALL)) {
// System.out.println(fbd.getBasins().keySet().size()
// + " rows in the table");
if ((centeredAggregationKey == null) || huc.equals(FFMPRecord.ALL)) {
// System.out.println(fbd.getBasins().keySet().size()
// + " rows in the table");
if (huc.equals(FFMPRecord.ALL)) {
FFMPBasinData fbd = baseRec.getBasinData();
tData = new FFMPTableData(fbd.getBasins().size());
for (Long key : fbd.getBasins().keySet()) {
if (huc.equals(FFMPRecord.ALL)) {
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb == null) {
continue;
}
for (DomainXML domain : domains) {
String cwa = domain.getCwa();
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb == null) {
continue;
}
if ((cwa.equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
try {
setFFMPRow(fbd.get(key), tData, false,
cwa);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(
virtualBasin.get(id),
tData, true, cwa);
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Couldn't create table row"
+ e);
}
}
}
for (DomainXML domain : domains) {
String cwa = domain.getCwa();
if ((cwa.equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
try {
setFFMPRow(fbd.get(key), tData, false, cwa);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
}
} else {
/*
* make sure at least one basin in the agg is in the
* CWA
*/
List<Long> pfafs = ft.getAggregatePfafs(key,
siteKey, huc);
boolean isVGB = false;
if (ft.checkVGBsInAggregate(key, siteKey, huc)) {
isVGB = true;
}
if (!pfafs.isEmpty()) {
FFMPBasinMetaData fmdb = ft.getBasinInDomains(
siteKey, domains, pfafs);
if (fmdb != null) {
try {
setFFMPRow(fbd.get(key), tData, isVGB,
null);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(virtualBasin.get(id),
tData, true, cwa);
} catch (Exception e) {
statusHandler.handle(
Priority.PROBLEM,
"Couldn't create table row"
+ e);
}
}
}
}
}
}
}
// show pfafs in aggregation
else {
for (Long key : resource.getCenteredAggregatePfafs()) {
// Find all of the basins for this HUC level
List<Long> keyList = ft
.getHucKeyList(siteKey, huc, domains);
tData = new FFMPTableData(keyList.size());
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
for (Long key : keyList) {
if (fmdb != null) {
for (DomainXML domain : domains) {
List<Long> pfafs = ft.getAggregatePfafs(key, siteKey,
huc);
if ((domain.getCwa().equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
boolean isVGB = false;
if (ft.checkVGBsInAggregate(key, siteKey, huc)) {
isVGB = true;
}
setFFMPRow(fbd.get(key), tData, false, null);
if (!pfafs.isEmpty()) {
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, huc,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(
virtualBasin.get(id),
tData, true, null);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row",
e);
}
FFMPBasinMetaData fmdb = ft.getBasinInDomains(
siteKey, domains, pfafs);
if (fmdb != null) {
try {
FFMPBasin basin = new FFMPBasin(key, true);
setFFMPRow(basin, tData, isVGB, null);
} catch (Exception e) {
statusHandler.handle(Priority.PROBLEM,
"Couldn't create table row", e);
}
}
}
}
}
}
// show pfafs in aggregation
else {
FFMPBasinData fbd = baseRec.getBasinData();
tData = new FFMPTableData(resource.getCenteredAggregatePfafs().size());
for (Long key : resource.getCenteredAggregatePfafs()) {
FFMPBasinMetaData fmdb = ft.getBasin(siteKey, key);
if (fmdb != null) {
for (DomainXML domain : domains) {
if ((domain.getCwa().equals(fmdb.getCwa()))
|| (domain.isPrimary() && fmdb
.isPrimaryCwa())) {
setFFMPRow(fbd.get(key), tData, false, null);
if (virtualBasin != null) {
for (Long id : ft
.getVirtualGageBasinLookupIds(
siteKey, key, FFMPRecord.ALL,
resource.basinTableDlg
.getRowName())) {
try {
setFFMPRow(virtualBasin.get(id),
tData, true, null);
} catch (Exception e) {
statusHandler
.handle(Priority.PROBLEM,
"Couldn't create table row",
e);
}
}
}
@ -333,7 +339,6 @@ public class FFMPDataGenerator {
Date tableTime = resource.getTableTime();
FIELDS field = null;
String localHuc = null;
FfmpTableConfigData ffmpTableCfgData = FfmpTableConfig.getInstance()
.getTableConfigData(siteKey);
@ -358,37 +363,27 @@ public class FFMPDataGenerator {
monitor.setQpeWindow(new FFMPTimeWindow(tableTime, qpeTime));
if (isWorstCase || (centeredAggregationKey != null)) {
// make sure that "ALL" is loaded
localHuc = FFMPRecord.ALL;
} else {
localHuc = huc;
}
FFMPRecord rateRecord = monitor.getRateRecord(product, siteKey,
dataKey, product.getRate(), paintRefTime, localHuc, true);
dataKey, product.getRate(), paintRefTime, true);
FFMPRecord qpeRecord = monitor.getQPERecord(product, siteKey, dataKey,
product.getQpe(), tableTime, localHuc, true);
product.getQpe(), tableTime, true);
FFMPRecord qpfRecord = monitor.getQPFRecord(product, siteKey, dataKey,
null, paintRefTime, localHuc, true);
null, paintRefTime, true);
guidRecords = monitor.getGuidanceRecords(product, siteKey, tableTime,
localHuc, true);
FFMPRecord virtualRecord = null;
if (localHuc.equals(FFMPRecord.ALL)) {
virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey,
product.getVirtual(), tableTime, localHuc, true);
}
true);
FFMPRecord virtualRecord = monitor.getVirtualRecord(product, siteKey, dataKey,
product.getVirtual(), tableTime, true);
try {
if (rateRecord != null) {
rateBasin = rateRecord.getBasinData(localHuc);
rateBasin = rateRecord.getBasinData();
if (!rateBasin.getBasins().isEmpty()) {
field = FIELDS.RATE;
baseRec = rateRecord;
}
}
if (qpeRecord != null) {
qpeBasin = qpeRecord.getBasinData(localHuc);
qpeBasin = qpeRecord.getBasinData();
if (!qpeBasin.getBasins().isEmpty()) {
field = FIELDS.QPE;
if (baseRec == null) {
@ -397,21 +392,21 @@ public class FFMPDataGenerator {
}
}
if (qpfRecord != null) {
qpfBasin = qpfRecord.getBasinData(localHuc);
qpfBasin = qpfRecord.getBasinData();
}
if (guidRecords != null) {
guidBasins = new HashMap<String, FFMPBasinData>();
for (String type : guidRecords.keySet()) {
if (guidRecords.get(type) != null) {
guidBasins.put(type, guidRecords.get(type)
.getBasinData(localHuc));
.getBasinData());
} else {
guidBasins.put(type, null);
}
}
}
if (virtualRecord != null) {
virtualBasin = virtualRecord.getBasinData(localHuc);
virtualBasin = virtualRecord.getBasinData();
}
// Get interpolators

187
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java
View file

@ -169,6 +169,8 @@ import com.vividsolutions.jts.geom.Point;
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen No longer schedules load threads,
* refactored updates
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
*
* </pre>
*
@ -598,15 +600,12 @@ public class FFMPResource extends
private FFMPBasin getBasin(Long key, FFMPRecord.FIELDS bfield,
Date recentTime, boolean aggregate) throws VizException {
FFMPBasin basin = null;
String huc = null;
if (aggregate) {
huc = getHuc();
basin = new FFMPBasin(key, aggregate);
} else {
huc = FFMPRecord.ALL;
basin = getRecord(bfield, recentTime).getBasinData().getBasins()
.get(key);
}
basin = getRecord(bfield, recentTime).getBasinData(huc).getBasins()
.get(key);
return basin;
}
@ -680,14 +679,14 @@ public class FFMPResource extends
break;
}
case RATE: {
value = getRateRecord(recentTime).getBasinData(
FFMPRecord.ALL).getMaxValue(pfafs, recentTime);
value = getRateRecord(recentTime).getBasinData()
.getMaxValue(pfafs, recentTime);
break;
}
case QPF: {
value = getQpfRecord(recentTime).getBasinData(
FFMPRecord.ALL).getAverageMaxValue(pfafs,
recentTime, getQpfSourceExpiration());
value = getQpfRecord(recentTime).getBasinData()
.getAverageMaxValue(pfafs, recentTime,
getQpfSourceExpiration());
break;
}
case GUIDANCE: {
@ -695,21 +694,17 @@ public class FFMPResource extends
.getCountyFipsByPfaf(pfafs.get(0));
value = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getMaxGuidanceValue(pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration(getFFGName()), fips);
break;
}
case QPE: {
value = getQpeRecord().getBasinData(FFMPRecord.ALL)
.getAccumMaxValue(
pfafs,
recentTime,
getTableTime(),
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
value = getQpeRecord().getBasinData().getAccumMaxValue(
pfafs, recentTime, getTableTime(),
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML().isRate());
break;
}
}
@ -737,32 +732,64 @@ public class FFMPResource extends
break;
}
case RATE:
value = getBasin(key, field, recentTime, aggregate)
.getValue(recentTime);
if (aggregate) {
value = getRateRecord(recentTime).getBasinData()
.getAverageValue(pfafs, recentTime);
} else {
value = getBasin(key, field, recentTime, aggregate)
.getValue(recentTime);
}
break;
case QPF: {
value = getBasin(key, field, recentTime, aggregate)
.getAverageValue(recentTime,
getQpfSourceExpiration());
if (aggregate) {
value = getQpfRecord(recentTime).getBasinData()
.getAverageValue(pfafs, recentTime,
getQpfSourceExpiration());
} else {
value = getBasin(key, field, recentTime, aggregate)
.getAverageValue(recentTime,
getQpfSourceExpiration());
}
break;
}
case GUIDANCE: {
value = getGuidanceValue(
(FFMPGuidanceBasin) getBasin(key, field,
recentTime, aggregate), recentTime,
getFFGName());
if (aggregate) {
getGuidanceRecord()
.getBasinData()
.getAverageGuidanceValue(
pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration(getFFGName()));
} else {
value = getGuidanceValue(
(FFMPGuidanceBasin) getBasin(key, field,
recentTime, aggregate), recentTime,
getFFGName());
}
break;
}
case QPE: {
value = getBasin(key, field, recentTime, aggregate)
.getAccumValue(
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
if (aggregate) {
value = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData()
.getPrimarySourceXML()
.isRate());
} else {
value = getBasin(key, field, recentTime, aggregate)
.getAccumValue(
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData()
.getPrimarySourceXML()
.isRate());
}
break;
}
}
@ -810,17 +837,15 @@ public class FFMPResource extends
boolean forced = forceResult.isForced();
if ((forcedPfafs.size() > 0) && forced) {
// Recalculate the guidance using the forced value(s)
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(ffgType),
new Float(value), forcedPfafs,
getGuidSourceExpiration(ffgType));
value = guidRecord.getBasinData().getAverageGuidanceValue(
pfafList, this.getGuidanceInterpolation(ffgType),
new Float(value), forcedPfafs,
getGuidSourceExpiration(ffgType));
} else if (forcedPfafs.size() > 0) {
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(ffgType),
Float.NaN, forcedPfafs,
getGuidSourceExpiration(ffgType));
value = guidRecord.getBasinData().getAverageGuidanceValue(
pfafList, this.getGuidanceInterpolation(ffgType),
Float.NaN, forcedPfafs,
getGuidSourceExpiration(ffgType));
}
}
@ -906,10 +931,8 @@ public class FFMPResource extends
if ((rateRecord == null) && isNewRate) {
try {
String huc = getHucIfWorstCase();
rateRecord = monitor.getRateRecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), recentTime, huc,
false);
getDataKey(), getPrimarySource(), recentTime, false);
isNewRate = false;
} catch (Exception e) {
e.printStackTrace();
@ -926,10 +949,9 @@ public class FFMPResource extends
public FFMPRecord getQpeRecord() {
try {
if ((qpeRecord == null) && (getTableTime() != null) && isNewQpe) {
String huc = getHucIfWorstCase();
qpeRecord = monitor.getQPERecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), getTableTime(), huc,
false);
qpeRecord = monitor
.getQPERecord(getProduct(), getSiteKey(), getDataKey(),
getPrimarySource(), getTableTime(), false);
isNewQpe = false;
}
} catch (Exception e) {
@ -992,9 +1014,8 @@ public class FFMPResource extends
}
}
String huc = getHucIfWorstCase();
qpfRecord = monitor.getQPFRecord(getProduct(), getSiteKey(),
getDataKey(), getPrimarySource(), date, huc, false);
getDataKey(), getPrimarySource(), date, false);
isNewQpf = false;
}
} catch (Exception e) {
@ -1014,7 +1035,7 @@ public class FFMPResource extends
if ((virtualRecord == null) && isNewVirtual) {
virtualRecord = monitor.getVirtualRecord(getProduct(),
getSiteKey(), getDataKey(), getPrimarySource(),
getTableTime(), FFMPRecord.ALL, false);
getTableTime(), false);
isNewVirtual = false;
}
@ -1801,13 +1822,13 @@ public class FFMPResource extends
List<Float> guids = null;
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpes = getQpeRecord().getBasinData(FFMPRecord.ALL)
qpes = getQpeRecord().getBasinData()
.getAccumValues(pfafs, getTableTime(),
recentTime, getQpeSourceExpiration(),
isRate());
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getGuidanceValues(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
@ -1818,20 +1839,19 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(getHuc())
.get(key)
.getAccumValue(
qpe = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, ffgType);
guid = getGuidanceRecord().getBasinData()
.getAverageGuidanceValue(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
diff = FFMPUtils.getDiffValue(qpe, guid);
}
@ -1839,14 +1859,14 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.get(key)
.getAccumValue(getTableTime(), recentTime,
getQpeSourceExpiration(), isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
.getBasinData().get(key),
recentTime, ffgType);
guid = forceValue(pfafs,
getBasin(key, getField(), recentTime, aggregate),
@ -1881,14 +1901,14 @@ public class FFMPResource extends
List<Float> qpes = null;
List<Float> guids = null;
if (getQpeRecord() != null) {
qpes = getQpeRecord().getBasinData(FFMPRecord.ALL)
qpes = getQpeRecord().getBasinData()
.getAccumValues(pfafs, getTableTime(),
recentTime, getQpeSourceExpiration(),
isRate());
}
if (getGuidanceRecord() != null) {
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getGuidanceValues(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
@ -1899,32 +1919,31 @@ public class FFMPResource extends
} else {
if ((getQpeRecord() != null)
&& (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(getHuc())
.get(key)
.getAccumValue(
qpe = getQpeRecord().getBasinData()
.getAccumAverageValue(
pfafs,
getTableTime(),
recentTime,
getQpeSourceExpiration(),
getResourceData().getPrimarySourceXML()
.isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
guid = getGuidanceRecord().getBasinData()
.getAverageGuidanceValue(pfafs,
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
}
}
} else {
if ((getQpeRecord() != null) && (getGuidanceRecord() != null)) {
qpe = getQpeRecord()
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.get(key)
.getAccumValue(getTableTime(), recentTime,
getQpeSourceExpiration(), isRate());
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
.getBasinData().get(key),
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
}
@ -4043,5 +4062,5 @@ public class FFMPResource extends
}
return dataTimes;
}
}

69
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResourceData.java
View file

@ -32,7 +32,6 @@ import javax.xml.bind.annotation.XmlType;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.IJobChangeEvent;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.core.runtime.jobs.JobChangeAdapter;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
@ -81,6 +80,7 @@ import com.raytheon.uf.viz.monitor.ffmp.xml.FFMPConfigBasinXML;
* Mar 06, 2013 1769 dhladky Changed threading to use count down latch.
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jun 06, 2013 2075 njensen Use new load jobs
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -208,7 +208,6 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
}
}
}
this.domains = defaults;
final Date mostRecentTime = availableTimes[availableTimes.length - 1]
@ -216,56 +215,23 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
final double configTimeFrame = cfgBasinXML.getTimeFrame();
final Date timeBack = new Date(
(long) (mostRecentTime.getTime() - (configTimeFrame * TimeUtil.MILLIS_PER_HOUR)));
final List<String> initialHucs = new ArrayList<String>();
initialHucs.add(FFMPRecord.ALL);
final String defaultLayer = monitor.getConfig()
.getFFMPConfigData().getLayer();
if (!defaultLayer.equals(FFMPRecord.ALL)) {
initialHucs.add(defaultLayer);
}
final List<String> onlyAllHuc = new ArrayList<String>();
onlyAllHuc.add(FFMPRecord.ALL);
InitialLoadJob initialJob = new InitialLoadJob(this, timeBack,
mostRecentTime, initialHucs);
mostRecentTime, onlyAllHuc);
// schedule the secondary load to start as soon as the initial
// completes
// secondary load will be the same time period as initial with
// the hucs that the initial job did not do
// schedule the background load to start as soon as the initial
// completes, it will load the other hours back to 24
initialJob.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
Date secondStartTime = timeBack;
List<String> secondaryHucs = FFMPTemplateConfigurationManager
.getInstance().getHucLevels();
secondaryHucs.removeAll(initialHucs);
BackgroundLoadJob secondaryJob = new BackgroundLoadJob(
"Secondary FFMP Load", FFMPResourceData.this,
secondStartTime, mostRecentTime, secondaryHucs);
secondaryJob.setPriority(Job.SHORT);
// schedule the tertiary load as soon as the
// secondary completes
// tertiary load will do 24 hours back of the
// same hucs as the initial load
secondaryJob
.addJobChangeListener(new JobChangeAdapter() {
@Override
public void done(IJobChangeEvent event) {
List<String> tertiaryHucs = new ArrayList<String>();
tertiaryHucs.add(FFMPRecord.ALL);
Date tertiaryStartTime = new Date(
mostRecentTime.getTime()
- (24 * TimeUtil.MILLIS_PER_HOUR));
BackgroundLoadJob tertiaryJob = new BackgroundLoadJob(
"Tertiary FFMP Load",
FFMPResourceData.this,
tertiaryStartTime, timeBack,
tertiaryHucs);
tertiaryJob
.setPreloadAvailableUris(true);
tertiaryJob.schedule();
}
});
secondaryJob.schedule();
Date backgroundStartTime = new Date(mostRecentTime
.getTime() - (24 * TimeUtil.MILLIS_PER_HOUR));
BackgroundLoadJob backgroundJob = new BackgroundLoadJob(
"Background FFMP Load", FFMPResourceData.this,
backgroundStartTime, timeBack, onlyAllHuc);
backgroundJob.setPreloadAvailableUris(true);
backgroundJob.schedule();
}
});
initialJob.schedule();
@ -274,7 +240,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
// background so the first paints of the resource
// will be faster
List<String> earlyLoadHucs = new ArrayList<String>();
earlyLoadHucs.addAll(initialHucs);
earlyLoadHucs.addAll(onlyAllHuc);
for (String otherHuc : FFMPTemplateConfigurationManager
.getInstance().getHucLevels()) {
if (!earlyLoadHucs.contains(otherHuc)) {
@ -324,8 +290,7 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
.getAvailableUris(siteKey, dataKey, sourceName,
standAloneTime);
monitor.processUris(sourceURIs, siteKey, sourceName,
standAloneTime, FFMPRecord.ALL,
SubMonitor.convert(null));
standAloneTime, SubMonitor.convert(null));
}
}
}
@ -427,10 +392,10 @@ public class FFMPResourceData extends AbstractRequestableResourceData {
public void populateRecord(FFMPRecord precord) throws VizException {
try {
getMonitor().populateFFMPRecord(siteKey, precord,
precord.getSourceName(), huc);
precord.getSourceName());
} catch (Exception e) {
throw new VizException("Failed to populate ffmp record "
+ precord.getDataURI() + " for huc " + huc);
+ precord.getDataURI());
}
}

188
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPRowGenerator.java
View file

@ -59,6 +59,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FfmpTableConfigData;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jun 11, 2013 2085 njensen Initial creation
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -208,7 +209,7 @@ public class FFMPRowGenerator implements Runnable {
new FFMPTableCellData(rowField, sb.toString(),
mouseOverText));
if (!isWorstCase || huc.equals(FFMPRecord.ALL)
if (huc.equals(FFMPRecord.ALL)
|| (centeredAggregationKey != null)) {
if (!cBasin.getValues().isEmpty()) {
@ -272,8 +273,73 @@ public class FFMPRowGenerator implements Runnable {
FIELDS.DIFF, diffValue));
i += 3;
}
} else {
} else if (isWorstCase) {
trd = getMaxValue(trd, cBasin);
} else {
// general Aggregate HUC processing
ArrayList<Long> pfafs = ft.getAggregatePfafs(
cBasin.getPfaf(), siteKey, domain);
if (!cBasin.getValues().isEmpty()) {
rate = vgBasin.getAverageValue(paintRefTime,
expirationTime);
if (sliderTime > 0.00) {
FFMPTimeWindow window = monitor.getQpeWindow();
qpeBasin.getAccumAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
} else {
qpe = 0.0f;
}
}
trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE,
rate));
trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE,
qpe));
if (qpfBasin != null) {
FFMPTimeWindow window = monitor.getQpfWindow();
qpf = qpfBasin.getAverageValue(pfafs,
window.getAfterTime(), window.getBeforeTime());
}
trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF,
qpf));
// run over each guidance type
int i = 0;
for (String guidType : guidBasins.keySet()) {
guidance = Float.NaN;
FFMPTableCellData guidCellData = getGuidanceCellData(
cBasin, domain, guidType, parentBasinPfaf);
if (guidCellData == null) {
// check for forcing even if no data are available
guidance = getForcedAvg(domain, cBasin, guidType);
boolean forced = !guidance.isNaN();
guidCellData = new FFMPTableCellData(
FIELDS.GUIDANCE, guidance, forced);
} else {
guidance = guidCellData.getValueAsFloat();
}
trd.setTableCellData(i + 4, guidCellData);
float ratioValue = Float.NaN;
float diffValue = Float.NaN;
// If guidance is NaN then it cannot be > 0
if (!qpe.isNaN() && (guidance > 0.0f)) {
ratioValue = FFMPUtils.getRatioValue(qpe, guidance);
diffValue = FFMPUtils.getDiffValue(qpe, guidance);
}
trd.setTableCellData(i + 5, new FFMPTableCellData(
FIELDS.RATIO, ratioValue));
trd.setTableCellData(i + 6, new FFMPTableCellData(
FIELDS.DIFF, diffValue));
i += 3;
}
}
trd.setSortCallback(tData);
@ -292,22 +358,46 @@ public class FFMPRowGenerator implements Runnable {
if (!isWorstCase || huc.equals(FFMPRecord.ALL)
|| (centeredAggregationKey != null)) {
ArrayList<Long> pfafs = null;
if (cBasin.getAggregated()) {
pfafs = ft.getAggregatePfafs(cBasin.getPfaf(), siteKey,
huc);
}
if (rateBasin != null) {
FFMPBasin basin = rateBasin.get(cBasinPfaf);
if (basin != null) {
rate = basin.getValue(paintRefTime);
if (cBasin.getAggregated()) {
rate = rateBasin.getAverageValue(pfafs,
paintRefTime);
} else {
FFMPBasin basin = rateBasin.get(cBasinPfaf);
if (basin != null) {
rate = basin.getValue(paintRefTime);
}
}
}
trd.setTableCellData(1, new FFMPTableCellData(FIELDS.RATE,
rate));
if (qpeBasin != null) {
FFMPBasin basin = qpeBasin.get(cBasinPfaf);
if (basin != null) {
FFMPTimeWindow window = monitor.getQpeWindow();
qpe = basin.getAccumValue(window.getAfterTime(),
FFMPTimeWindow window = monitor.getQpeWindow();
if (cBasin.getAggregated()) {
qpe = qpeBasin.getAccumAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
} else {
FFMPBasin basin = qpeBasin.get(cBasinPfaf);
if (basin != null) {
qpe = basin.getAccumValue(
window.getAfterTime(),
window.getBeforeTime(), expirationTime,
isRate);
}
}
}
@ -315,11 +405,19 @@ public class FFMPRowGenerator implements Runnable {
qpe));
if (qpfBasin != null) {
FFMPBasin basin = qpfBasin.get(cBasinPfaf);
if (basin != null) {
FFMPTimeWindow window = monitor.getQpfWindow();
qpf = basin.getAverageValue(window.getAfterTime(),
FFMPTimeWindow window = monitor.getQpfWindow();
if (cBasin.getAggregated()) {
qpf = qpfBasin.getAverageValue(pfafs,
window.getAfterTime(),
window.getBeforeTime());
} else {
FFMPBasin basin = qpfBasin.get(cBasinPfaf);
if (basin != null) {
qpf = basin.getAverageValue(
window.getAfterTime(),
window.getBeforeTime());
}
}
}
@ -436,13 +534,13 @@ public class FFMPRowGenerator implements Runnable {
if (cBasin instanceof FFMPVirtualGageBasin) {
if (!pfafs.isEmpty()) {
if (virtualBasin != null) {
rate = virtualBasin.get(cBasin.getPfaf()).getValue(
paintRefTime);
rate = virtualBasin.getMaxValue(pfafs, paintRefTime);
if (sliderTime > 0.00) {
qpe = virtualBasin.get(cBasin.getPfaf()).getAccumValue(
monitor.getQpeWindow().getAfterTime(),
monitor.getQpeWindow().getBeforeTime(),
qpe = virtualBasin.getAccumMaxValue(pfafs, monitor
.getQpeWindow().getAfterTime(), monitor
.getQpeWindow().getBeforeTime(),
expirationTime, isRate);
} else {
qpe = 0.0f;
@ -454,10 +552,9 @@ public class FFMPRowGenerator implements Runnable {
trd.setTableCellData(2, new FFMPTableCellData(FIELDS.QPE, qpe));
if (qpfBasin != null) {
qpf = new Float(qpfBasin.get(cBasin.getPfaf()).getMaxValue(
monitor.getQpfWindow().getAfterTime(),
monitor.getQpfWindow().getBeforeTime()))
.floatValue();
qpf = qpfBasin.getMaxValue(pfafs, monitor.getQpfWindow()
.getAfterTime(), monitor.getQpfWindow()
.getBeforeTime());
}
trd.setTableCellData(3, new FFMPTableCellData(FIELDS.QPF, qpf));
@ -534,7 +631,7 @@ public class FFMPRowGenerator implements Runnable {
if (isWorstCase) {
guidance = guidRecords
.get(guidType)
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getMaxGuidanceValue(
pfafs,
resource.getGuidanceInterpolators()
@ -542,11 +639,16 @@ public class FFMPRowGenerator implements Runnable {
resource.getGuidSourceExpiration(guidType),
cBasin.getPfaf());
} else {
FFMPGuidanceBasin basin = (FFMPGuidanceBasin) guidRecords
.get(guidType).getBasinData(huc)
.get(cBasin.getPfaf());
guidance = resource.getGuidanceValue(basin, monitor
.getQpeWindow().getBeforeTime(), guidType);
guidance = guidRecords
.get(guidType)
.getBasinData()
.getAverageGuidanceValue(
pfafs,
resource.getGuidanceInterpolators()
.get(guidType),
resource.getGuidSourceExpiration(guidType));
}
trd.setTableCellData(i + 4, new FFMPTableCellData(
@ -766,17 +868,6 @@ public class FFMPRowGenerator implements Runnable {
private FFMPTableCellData getGuidanceCellData(FFMPBasin cBasin,
String domain, String guidType, Long parentBasinPfaf) {
long cBasinPfaf = cBasin.getPfaf();
FFMPBasinData guidBasin = guidBasins.get(guidType);
FFMPGuidanceBasin ffmpGuidBasin = null;
if (guidBasin != null) {
ffmpGuidBasin = (FFMPGuidanceBasin) guidBasin.get(cBasinPfaf);
}
if (ffmpGuidBasin == null) {
return null;
}
List<Long> pfafList = Collections.emptyList();
List<Long> forcedPfafs = Collections.emptyList();
boolean forced = false;
@ -806,8 +897,7 @@ public class FFMPRowGenerator implements Runnable {
if (FFFGDataMgr.getInstance().isForcingConfigured()) {
FFMPBasin parentBasin = cBasin;
if (cBasinPfaf != parentBasinPfaf.longValue()) {
parentBasin = baseRec.getBasinData(FFMPRecord.ALL).get(
parentBasinPfaf);
parentBasin = baseRec.getBasinData().get(parentBasinPfaf);
}
ForceUtilResult forceResult = forceUtil.calculateForcings(domain,
ft, parentBasin);
@ -819,20 +909,22 @@ public class FFMPRowGenerator implements Runnable {
// Recalculate guidance using the forced value(s)
guidance = guidRecords
.get(guidType)
.getBasinData(FFMPRecord.ALL)
.getBasinData()
.getAverageGuidanceValue(pfafList,
resource.getGuidanceInterpolators().get(guidType),
guidance, forcedPfafs,
resource.getGuidSourceExpiration(guidType));
} else {
if (ffmpGuidBasin != null) {
guidance = resource.getGuidanceValue(ffmpGuidBasin,
paintRefTime, guidType);
if (guidance < 0.0f) {
guidance = Float.NaN;
}
FFMPGuidanceBasin ffmpGuidBasin = (FFMPGuidanceBasin) guidRecords
.get(guidType).getBasinData().get(cBasinPfaf);
guidance = resource.getGuidanceValue(ffmpGuidBasin, paintRefTime,
guidType);
if (guidance < 0.0f) {
guidance = Float.NaN;
}
}
return new FFMPTableCellData(FIELDS.GUIDANCE, guidance, forced);

51
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/thread/AbstractLoadJob.java
View file

@ -22,7 +22,6 @@ package com.raytheon.uf.viz.monitor.ffmp.ui.thread;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
@ -31,7 +30,6 @@ import java.util.Set;
import org.eclipse.core.runtime.SubMonitor;
import org.eclipse.core.runtime.jobs.Job;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.xml.FFMPRunXML;
import com.raytheon.uf.common.monitor.xml.ProductRunXML;
@ -54,6 +52,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.rsc.FFMPResourceData;
* ------------ ---------- ----------- --------------------------
* Jun 04, 2013 2075 njensen Initial creation
* Jun 07, 2013 2075 njensen Added progress monitoring
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -81,8 +80,6 @@ public abstract class AbstractLoadJob extends Job {
protected FFMPMonitor ffmpMonitor;
protected List<String> hucsToLoad = null;
/**
* Constructor
*
@ -105,28 +102,13 @@ public abstract class AbstractLoadJob extends Job {
this.resourceData = resourceData;
this.startTime = timeBack;
this.endTime = mostRecentTime;
this.hucsToLoad = hucsToLoad;
// configure FFMP
this.hucsToLoad.remove(FFMPRecord.VIRTUAL);
FFMPRunXML runXML = FFMPRunConfigurationManager.getInstance()
.getRunner(resourceData.wfo);
this.productRun = runXML.getProduct(resourceData.siteKey);
this.product = resourceData.getProduct();
this.ffmpMonitor = FFMPMonitor.getInstance();
// just for debugging/logging
StringBuilder sb = new StringBuilder();
sb.append(name);
sb.append(" hucs to load: ");
Iterator<String> itr = this.hucsToLoad.iterator();
while (itr.hasNext()) {
sb.append(itr.next());
if (itr.hasNext()) {
sb.append(", ");
}
}
System.out.println(sb.toString());
}
/**
@ -165,10 +147,8 @@ public abstract class AbstractLoadJob extends Job {
}
}
if (rateURI != null) {
for (String phuc : hucsToLoad) {
ffmpMonitor.processUri(rateURI, resourceData.siteKey,
product.getRate(), startTime, phuc);
}
ffmpMonitor.processUri(rateURI, resourceData.siteKey,
product.getRate(), startTime);
}
}
@ -190,12 +170,10 @@ public abstract class AbstractLoadJob extends Job {
protected void doQpe(NavigableMap<Date, List<String>> qpeURIs,
SubMonitor smonitor) {
if (!qpeURIs.isEmpty()) {
smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR);
for (String phuc : hucsToLoad) {
ffmpMonitor.processUris(qpeURIs, resourceData.siteKey,
product.getQpe(), startTime, phuc,
smonitor.newChild(PROGRESS_FACTOR));
}
smonitor.beginTask(null, PROGRESS_FACTOR);
ffmpMonitor.processUris(qpeURIs, resourceData.siteKey,
product.getQpe(), startTime,
smonitor.newChild(PROGRESS_FACTOR));
}
}
@ -246,12 +224,11 @@ public abstract class AbstractLoadJob extends Job {
// Use this method of QPF data retrieval if you don't have cache
// files
if (!qpfURIs.isEmpty()) {
smonitor.beginTask(null, hucsToLoad.size() * PROGRESS_FACTOR);
for (String phuc : hucsToLoad) {
ffmpMonitor.processUris(qpfURIs, resourceData.siteKey,
productQpf, startTime, phuc,
smonitor.newChild(PROGRESS_FACTOR));
}
smonitor.beginTask(null, PROGRESS_FACTOR);
ffmpMonitor.processUris(qpfURIs, resourceData.siteKey, productQpf,
startTime, smonitor.newChild(PROGRESS_FACTOR));
}
}
@ -264,7 +241,7 @@ public abstract class AbstractLoadJob extends Job {
product.getVirtual(), startTime);
if (!virtualURIs.isEmpty()) {
ffmpMonitor.processUris(virtualURIs, resourceData.siteKey,
product.getVirtual(), startTime, FFMPRecord.ALL, smonitor);
product.getVirtual(), startTime, smonitor);
}
}
@ -303,7 +280,7 @@ public abstract class AbstractLoadJob extends Job {
if (iguidURIs != null && !iguidURIs.isEmpty()) {
ffmpMonitor.processUris(iguidURIs, resourceData.siteKey,
guidSource.getSourceName(), startTime,
FFMPRecord.ALL,
smonitor.newChild(PROGRESS_FACTOR / subWork));
}
}

42
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java
View file

@ -21,7 +21,6 @@ package com.raytheon.uf.common.dataplugin.ffmp;
**/
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.raytheon.uf.common.serialization.ISerializableObject;
@ -37,6 +36,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Created to reduce memory and disk read/writes for FFMP
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -52,8 +52,6 @@ public class FFMPAggregateRecord implements ISerializableObject {
@DynamicSerializeElement
private String sourceSiteDataKey;
private static final long serialVersionUID = 767745643535471L;
/**
* Aggregate Record implementation for FFMP
@ -63,8 +61,16 @@ public class FFMPAggregateRecord implements ISerializableObject {
}
@DynamicSerializeElement
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
private FFMPBasinData basins;
public FFMPBasinData getBasins() {
return basins;
}
public void setBasins(FFMPBasinData basins) {
this.basins = basins;
}
@DynamicSerializeElement
private List<Long> times = new ArrayList<Long>();
@ -83,34 +89,6 @@ public class FFMPAggregateRecord implements ISerializableObject {
public List<Long> getTimes() {
return times;
}
public void setBasinsMap(HashMap<String, FFMPBasinData> basinsMap) {
this.basinsMap = basinsMap;
}
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Add a basin Data aggregate object
* @param cacheData
*/
public void addBasinData(FFMPBasinData basinData) {
basinsMap.put(basinData.getHucLevel(), basinData);
}
/**
* Gets the BasinData object
* @param huc
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinsMap.containsKey(huc)) {
return basinsMap.get(huc);
}
return null;
}
/**
* WFO designator

2
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java
View file

@ -31,7 +31,6 @@ import javax.persistence.Transient;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.common.time.util.ImmutableDate;
/**
* FFMP basin/aggregated value holder
@ -48,6 +47,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* from aggregate records and delayed
* TreeMap creation to the tertiary loader.
* Apr 26, 2013 1954 bsteffen Minor code cleanup throughout FFMP.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*

94
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java
View file

@ -51,6 +51,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* 01/27/13 1569 D. Hladky Added support for write of aggregate record cache
* 04/16/13 1912 bsteffen Initial bulk hdf5 access for ffmp
* 05/09/13 1919 mpduff Use parent pfaf instead of lookupId.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -59,7 +60,6 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
*/
@DynamicSerialize
public class FFMPBasinData implements ISerializableObject {
private static final long serialVersionUID = 8162247989509750715L;
public static final double GUIDANCE_MISSING = -999999.0;
@ -80,15 +80,6 @@ public class FFMPBasinData implements ISerializableObject {
*/
private final Map<String, FFMPBasin[]> orderedBasinsCache = new HashMap<String, FFMPBasin[]>();
/**
* Public one arg constructor
*
* @param huc_level
*/
public FFMPBasinData(String hucLevel) {
setHucLevel(hucLevel);
}
/**
* No arg hibernate constructor
*/
@ -172,6 +163,8 @@ public class FFMPBasinData implements ISerializableObject {
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param startDate
* @param finishDate
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date beforeDate,
@ -191,6 +184,55 @@ public class FFMPBasinData implements ISerializableObject {
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param exact
* date
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date date) {
float tvalue = 0.0f;
int i = 0;
for (Long pfaf : pfaf_ids) {
FFMPBasin basin = getBasins().get(pfaf);
if (basin != null) {
tvalue += basin.getValue(date);
i++;
}
}
tvalue = tvalue / i;
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins
*
* @param pfaf_ids
* @param date
* @param expirationTime
* @return
*/
public float getAverageValue(ArrayList<Long> pfaf_ids, Date date,
long epirationTime) {
float tvalue = 0.0f;
int i = 0;
for (Long pfaf : pfaf_ids) {
FFMPBasin basin = getBasins().get(pfaf);
if (basin != null) {
tvalue += basin.getAverageValue(date, epirationTime);
i++;
}
}
tvalue = tvalue / i;
return tvalue;
}
/**
* Extracts the average value for an aggregation of basins with areal
* weighting
@ -229,8 +271,8 @@ public class FFMPBasinData implements ISerializableObject {
* @param pfaf_ids
* @return
*/
public float getAccumAverageValue(ArrayList<Long> pfaf_ids,
Date beforeDate, Date afterDate, long expirationTime, boolean rate) {
public float getAccumAverageValue(List<Long> pfaf_ids, Date beforeDate,
Date afterDate, long expirationTime, boolean rate) {
float tvalue = 0.0f;
int i = 0;
@ -534,6 +576,34 @@ public class FFMPBasinData implements ISerializableObject {
return values;
}
/**
* Gets the average guidance value for an aggregate basin
*
* @param pfaf_ids
* @param interpolation
* @param expiration
* @return
*/
public Float getAverageGuidanceValue(List<Long> pfaf_ids,
FFMPGuidanceInterpolation interpolation, long expiration) {
float tvalue = 0.0f;
int i = 0;
List<Float> vals = getGuidanceValues(pfaf_ids, interpolation,
expiration);
if (vals != null) {
for (Float val : vals) {
tvalue += val;
i++;
}
} else {
return null;
}
return tvalue / i;
}
/**
* used for max ratio and diff calcs
*

162
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java
View file

@ -24,8 +24,6 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager.SOURCE_TYPE;
import com.raytheon.uf.common.monitor.xml.SourceXML;
@ -47,6 +45,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException
* 01/27/13 1478 D. Hladky Re-worked to help with memory size and NAS read write stress
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -59,10 +58,10 @@ public class FFMPDataContainer {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPDataContainer.class);
private final Map<String, FFMPBasinData> basinDataMap = new ConcurrentHashMap<String, FFMPBasinData>();// DR
private FFMPBasinData basins;
private String sourceName = null;
private boolean isPurged = false;
public FFMPDataContainer() {
@ -71,40 +70,41 @@ public class FFMPDataContainer {
/**
* Usual constructor
*
* @param sourceName
*/
public FFMPDataContainer(String sourceName) {
this.sourceName = sourceName;
basinDataMap.put(FFMPRecord.ALL, new FFMPBasinData(FFMPRecord.ALL));
// System.out.println("Creating source: " + sourceName);
basins = new FFMPBasinData();
}
/**
* special constuctor
*
* @param sourceName
* @param hucs
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
basinDataMap.put(huc, new FFMPBasinData(huc));
}
basins = new FFMPBasinData();
}
/**
* new container first time read in from cache
*
* @param sourceName
* @param hucs
* @param record
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs, FFMPAggregateRecord record) {
public FFMPDataContainer(String sourceName, FFMPAggregateRecord record) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
FFMPBasinData basinData = record.getBasinData(huc);
basinData.populate(record.getTimes());
basinDataMap.put(huc, basinData);
}
FFMPBasinData basinData = record.getBasins();
basinData.populate(record.getTimes());
basins = basinData;
}
/**
@ -114,7 +114,7 @@ public class FFMPDataContainer {
* @param newBasinData
*/
public void addFFMPEntry(Date date, SourceXML source,
FFMPBasinData newBasinData, String huc, String siteKey) {
FFMPBasinData newBasinData, String siteKey) {
boolean guid = false;
@ -122,12 +122,12 @@ public class FFMPDataContainer {
guid = true;
}
FFMPBasinData currBasinData = getBasinData(huc);
FFMPBasinData currBasinData = getBasinData();
synchronized (currBasinData) {
if (currBasinData == null) {
setBasinData(huc, newBasinData);
setBasinData(newBasinData);
} else {
for (Long key : newBasinData.getBasins().keySet()) {
@ -158,7 +158,7 @@ public class FFMPDataContainer {
val);
}
//currBasinData.put(key, basin);
// currBasinData.put(key, basin);
syncPut(currBasinData, key, basin);
} else {
@ -243,7 +243,7 @@ public class FFMPDataContainer {
}
basin.setValue(date, val);
//currBasinData.put(key, basin);
// currBasinData.put(key, basin);
syncPut(currBasinData, key, basin);
} else {
@ -283,10 +283,9 @@ public class FFMPDataContainer {
public boolean containsKey(Date date) {
boolean contains = false;
if (getBasinData(FFMPRecord.ALL) != null) {
if (getBasinData() != null) {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -309,7 +308,7 @@ public class FFMPDataContainer {
*/
public boolean containsKey(String sourceName) {
boolean contains = false;
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL).getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -334,18 +333,14 @@ public class FFMPDataContainer {
*
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinDataMap.containsKey(huc)) {
return basinDataMap.get(huc);
public FFMPBasinData getBasinData() {
if (basins != null) {
return basins;
} else {
return null;
}
}
public Set<String> getKeys() {
return basinDataMap.keySet();
}
/**
* Get the maximum value in the monitored area.
*
@ -359,9 +354,8 @@ public class FFMPDataContainer {
public double getMaxValue(ArrayList<Long> pfafs, Date backDate,
Date currDate, long expirationTime, boolean rate) {
double val = getBasinData(FFMPRecord.ALL).getAccumMaxValue(pfafs,
currDate,
backDate, expirationTime, rate);
double val = getBasinData().getAccumMaxValue(pfafs, currDate, backDate,
expirationTime, rate);
return val;
}
@ -374,8 +368,7 @@ public class FFMPDataContainer {
public Date getNewest() {
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -402,8 +395,7 @@ public class FFMPDataContainer {
*/
public Date getOldest() {
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -431,8 +423,7 @@ public class FFMPDataContainer {
public List<Date> getOrderedTimes(Date barrierTime) {
ArrayList<Date> orderedTimes = new ArrayList<Date>();
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -453,16 +444,16 @@ public class FFMPDataContainer {
return null;
}
/**
* Gets the list of times for serialization
*
* @return
*/
public List<Long> getOrderedTimes() {
ArrayList<Long> orderedTimes = new ArrayList<Long>();
try {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL)
.getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -484,6 +475,7 @@ public class FFMPDataContainer {
/**
* Gets the source name for this Data Container
*
* @return
*/
public String getSourceName() {
@ -492,13 +484,13 @@ public class FFMPDataContainer {
/**
* Clean up old junk
*
* @param backDate
*/
public void purge(Date backDate) {
statusHandler.handle(Priority.INFO, "Purging "+getSourceName()+" Container back to: "+backDate);
for (String huc : basinDataMap.keySet()) {
getBasinData(huc).purgeData(backDate);
}
statusHandler.handle(Priority.INFO, "Purging " + getSourceName()
+ " Container back to: " + backDate);
getBasinData().purgeData(backDate);
setPurged(true);
}
@ -511,27 +503,25 @@ public class FFMPDataContainer {
// create a record from the cache record
FFMPRecord record = new FFMPRecord(cacheRecord);
for (Entry<String, FFMPBasinData> dentry : record.getBasinsMap()
.entrySet()) {
for (Entry<Long, FFMPBasin> entry : dentry.getValue().getBasins()
.entrySet()) {
FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
basin.getValues().putAll(entry.getValue().getValues());
}
FFMPBasinData dentry = record.getBasinData();
for (Entry<Long, FFMPBasin> entry : dentry.getBasins().entrySet()) {
FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
syncPut(getBasinData(dentry.getKey()), entry.getKey(),
entry.getValue());
basin.getValues().putAll(entry.getValue().getValues());
}
} else {
syncPut(getBasinData(), entry.getKey(), entry.getValue());
}
}
}
/**
@ -539,8 +529,8 @@ public class FFMPDataContainer {
*
* @param basins
*/
public void setBasinData(String huc, FFMPBasinData fftiData) {
basinDataMap.put(huc, fftiData);
public void setBasinData(FFMPBasinData basins) {
this.basins = basins;
}
public void setSourceName(String sourceName) {
@ -555,7 +545,7 @@ public class FFMPDataContainer {
*/
public int size() {
Map<Long, FFMPBasin> basins = getBasinData(FFMPRecord.ALL).getBasins();
Map<Long, FFMPBasin> basins = getBasinData().getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
@ -569,34 +559,27 @@ public class FFMPDataContainer {
}
return 0;
}
/**
* DR 15471 lock put() to avoid ConcurrentModificationException
*/
private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value){
if(fbd==null || key==null)
return;
Map<Long, FFMPBasin> basins = fbd.getBasins();
if(basins == null)
return;
synchronized (basins) {
basins.put(key, value);
}
}
/**
* Gets the basin data map
* @return
* DR 15471 lock put() to avoid ConcurrentModificationException
*/
public Map<String, FFMPBasinData> getBasinMap() {
return basinDataMap;
private void syncPut(FFMPBasinData fbd, Long key, FFMPBasin value) {
if (fbd == null || key == null)
return;
Map<Long, FFMPBasin> basins = fbd.getBasins();
if (basins == null)
return;
synchronized (basins) {
basins.put(key, value);
}
}
/**
* Sets whether this container has been purged or not
*
* @param isPurged
*/
public void setPurged(boolean isPurged) {
@ -605,6 +588,7 @@ public class FFMPDataContainer {
/**
* Has this container been purged?
*
* @return
*/
public boolean isPurged() {

115
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java
View file

@ -24,7 +24,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.lang.ref.WeakReference;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@ -89,6 +88,7 @@ import com.raytheon.uf.common.time.util.ImmutableDate;
* Apr 18, 2013 1919 dhladky Added method for VGB loading
* May 07, 2013 1869 bsteffen Remove dataURI column from
* PluginDataObject.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -142,23 +142,25 @@ public class FFMPRecord extends PersistablePluginDataObject
private String siteKey;
@Transient
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
private FFMPBasinData basins = new FFMPBasinData();;
@Transient
private int expiration = 0;
@Transient
private boolean isRate = false;
protected static ConcurrentMap<Long, WeakReference<ImmutableDate>> cacheTimes = new ConcurrentHashMap<Long, WeakReference<ImmutableDate>>();
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPRecord.class);
/** ALL HUC LEVEL **/
public static final String ALL = "ALL";
/** COUNTY HUC LEVEL **/
public static final String COUNTY = "COUNTY";
/** VIRTUAL HUC LEVEL **/
public static final String VIRTUAL = "VIRTUAL";
@ -359,8 +361,8 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param hucName
*/
public void setBasinData(FFMPBasinData basins, String hucName) {
basinsMap.put(hucName, basins);
public void setBasinData(FFMPBasinData basins) {
this.basins = basins;
}
/**
@ -369,24 +371,10 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param basins
* @param hucName
*/
public FFMPBasinData getBasinData(String hucName) {
FFMPBasinData basins = basinsMap.get(hucName);
if (basins == null) {
basins = new FFMPBasinData(hucName);
basinsMap.put(hucName, basins);
}
public FFMPBasinData getBasinData() {
return basins;
}
/**
* Gets the map if you need it
*
* @return
*/
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Gets the Hash out of the datastore by HUC
*
@ -394,41 +382,36 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param huc
*/
public void retrieveMapFromDataStore(File datastoreFile, String uri,
FFMPTemplates template, String huc, Date date, String sourceName)
FFMPTemplates template, Date date, String sourceName)
throws Exception {
FFMPBasinData fbd = getBasinData(huc);
FFMPBasinData fbd = getBasinData();
ImmutableDate idate = getCacheDate(date);
boolean aggregate = true;
if (huc.equals(ALL)) {
aggregate = false;
}
boolean aggregate = false;
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(getSiteKey(),
domain.getCwa(), huc);
domain.getCwa(), FFMPRecord.ALL);
if (map != null && !map.isEmpty()) {
fbd.addBasins(datastoreFile, uri, getSiteKey(),
domain.getCwa(), huc, sourceName, idate, map.keySet(),
aggregate);
domain.getCwa(), FFMPRecord.ALL, sourceName, idate,
map.keySet(), aggregate);
}
}
}
public void retrieveMapFromDataStore(FFMPTemplates template, String huc)
public void retrieveMapFromDataStore(FFMPTemplates template)
throws Exception {
retrieveMapFromDataStore(getDataStoreFile(), getDataURI(), template,
huc, getDataTime().getRefTime(), getSourceName());
getDataTime().getRefTime(), getSourceName());
}
public void retrieveVirtualMapFromDataStore(FFMPTemplates template,
String huc) throws Exception {
retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(), template,
getDataTime().getRefTime(), getSourceName());
public void retrieveVirtualMapFromDataStore(FFMPTemplates template)
throws Exception {
retrieveVirtualMapFromDataStore(getDataStoreFile(), getDataURI(),
template, getDataTime().getRefTime(), getSourceName());
}
private File getDataStoreFile() {
@ -523,7 +506,7 @@ public class FFMPRecord extends PersistablePluginDataObject
public void retrieveVirtualMapFromDataStore(File datastoreFile, String uri,
FFMPTemplates template, Date date, String sourceName)
throws StorageException, FileNotFoundException {
FFMPBasinData fbd = getBasinData(ALL);
FFMPBasinData fbd = getBasinData();
String key = getDataKey();
ImmutableDate idate = getCacheDate(date);
@ -537,8 +520,7 @@ public class FFMPRecord extends PersistablePluginDataObject
if (size > 0) {
fbd.addVirtualBasins(datastoreFile, uri, key,
domain.getCwa(), idate,
lids.values());
domain.getCwa(), idate, lids.values());
}
}
}
@ -569,10 +551,9 @@ public class FFMPRecord extends PersistablePluginDataObject
public String toString() {
StringBuffer sb = new StringBuffer();
sb.append("\n dataURI: " + getDataURI() + "\n");
if (basinsMap != null) {
for (String key : basinsMap.keySet()) {
sb.append(key + " : " + basinsMap.get(key).getBasins().size()
+ "\n");
if (basins != null) {
for (Long key : basins.getBasins().keySet()) {
sb.append(key + " : " + basins.get(key).getValue() + "\n");
}
}
@ -635,10 +616,7 @@ public class FFMPRecord extends PersistablePluginDataObject
* @param date
*/
public void purgeData(Date date) {
for (FFMPBasinData basinData : getBasinsMap().values()) {
basinData.purgeData(date);
}
basins.purgeData(date);
}
public void setSiteKey(String siteKey) {
@ -648,21 +626,20 @@ public class FFMPRecord extends PersistablePluginDataObject
public String getSiteKey() {
return siteKey;
}
/**
* Get the fully populated aggregate record
*
* @return
*/
public FFMPAggregateRecord getAggregateRecord() {
FFMPAggregateRecord fdcr = new FFMPAggregateRecord();
for (FFMPBasinData basinData: basinsMap.values()) {
fdcr.addBasinData(basinData);
}
fdcr.setBasins(basins);
return fdcr;
}
/**
* Creates and populates a version of this record from an aggregate record
*
@ -671,14 +648,11 @@ public class FFMPRecord extends PersistablePluginDataObject
public FFMPRecord(FFMPAggregateRecord fdcr) {
List<Long> times = fdcr.getTimes();
for (FFMPBasinData basinData: fdcr.getBasinsMap().values()) {
// Keep in mind times can be null, Guidance basins are like that
basinData.populate(times);
setBasinData(basinData, basinData.getHucLevel());
}
FFMPBasinData fdcrBasins = fdcr.getBasins();
fdcrBasins.populate(times);
setBasinData(fdcrBasins);
}
/**
* Gets and maintains the list of times. This will lesson memory consumption
* because it means all FFMPBasin TreeMap date keys reference back to this
@ -691,7 +665,7 @@ public class FFMPRecord extends PersistablePluginDataObject
WeakReference<ImmutableDate> idate = cacheTimes.get(date.getTime());
ImmutableDate myDate = null;
if (idate != null) {
myDate = idate.get();
}
@ -701,22 +675,19 @@ public class FFMPRecord extends PersistablePluginDataObject
myDate = new ImmutableDate(time);
idate = new WeakReference<ImmutableDate>(myDate);
cacheTimes.putIfAbsent(time, idate);
}
}
return myDate;
}
/**
* Populate data from the cache files
*
* @param basins
* @param hucName
*/
public void populate(FFMPBasinData basins, String hucName) {
setBasinData(basins, hucName);
//System.out.println("Adding Whole Object Cache Data: "+hucName+" "+getSourceName());
public void populate(FFMPBasinData basins) {
setBasinData(basins);
}
@Override

48
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java
View file

@ -88,6 +88,7 @@ import com.vividsolutions.jts.io.WKBReader;
* 03/18/13 1817 D. Hladky Fixed issue with BOX where only 1 HUC was showing up.
* 04/15/13 1902 M. Duff Generic List
* 06/10/13 2085 njensen Use countyMap for efficiency
* 07/15/13 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -1310,6 +1311,25 @@ public class FFMPTemplates {
return map;
}
/**
* Find the list of pfafs for this HUC level
* @param siteKey
* @param huc
* @param domains
* @return
*/
public synchronized List<Long> getHucKeyList(String siteKey, String huc, List<DomainXML> domains) {
Set<Long> keys = new HashSet<Long>();
for (DomainXML domain: domains) {
LinkedHashMap<Long, ?> map = getMap(siteKey, domain.getCwa(), huc);
keys.addAll(map.keySet());
}
return new ArrayList<Long>(keys);
}
/**
* Gets the template config manager
@ -1663,23 +1683,28 @@ public class FFMPTemplates {
}
HashMap<String, HashMap<Long, ArrayList<FFMPVirtualGageBasinMetaData>>> virtualMap = virtualGageBasinsInParentPfaf
.get(dataKey);
ArrayList<Long> result = new ArrayList<Long>();
for (DomainXML domain : domains) {
HashMap<Long, ArrayList<FFMPVirtualGageBasinMetaData>> map = virtualMap
.get(domain.getCwa());
if (map != null) {
ArrayList<FFMPVirtualGageBasinMetaData> list = map.get(pfaf);
if (list != null && !list.isEmpty()) {
ArrayList<Long> result = new ArrayList<Long>();
for (FFMPVirtualGageBasinMetaData md : list)
result.add(md.getLookupId());
return result;
for (FFMPVirtualGageBasinMetaData md : list) {
if (!result.contains(md.getLookupId())) {
result.add(md.getLookupId());
}
}
}
}
}
return new ArrayList<Long>();
return result;
}
@ -2423,6 +2448,8 @@ public class FFMPTemplates {
HashMap<String, HashMap<String, ArrayList<FFMPVirtualGageBasinMetaData>>> virtualMap = vgbsInCounty
.get(dataKey);
ArrayList<Long> result = new ArrayList<Long>();
for (DomainXML domain : domains) {
HashMap<String, ArrayList<FFMPVirtualGageBasinMetaData>> map = virtualMap
@ -2432,16 +2459,17 @@ public class FFMPTemplates {
.get(stateCommaCnty.trim().toUpperCase());
if (list != null && !list.isEmpty()) {
ArrayList<Long> result = new ArrayList<Long>();
for (FFMPVirtualGageBasinMetaData md : list) {
result.add(md.getLookupId());
for (FFMPVirtualGageBasinMetaData md : list) {
if (!result.contains(md.getLookupId())) {
result.add(md.getLookupId());
}
}
return result;
}
}
}
return new ArrayList<Long>();
return result;
}
}

144
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dao/FFMPDao.java
View file

@ -53,6 +53,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 07/01/09 2521 dhladky Initial Creation
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -103,63 +104,36 @@ public class FFMPDao extends PluginDao {
// ignore data outside of domain
if (vmap.size() > 0) {
for (String key : record.getBasinsMap().keySet()) {
FFMPBasinData fbd = record.getBasinData(key);
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(), key);
int size = 0;
if (key.equals(FFMPRecord.ALL)) {
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
size++;
}
}
} else {
for (Long pfaf : map.keySet()) {
ArrayList<Long> vgbpfafs = template
.getVGBsInAggregate(pfaf,
record.getSiteKey(), key);
if (vgbpfafs != null) {
size++;
}
}
FFMPBasinData fbd = record.getBasinData();
int size = 0;
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
size++;
}
float[] dataRec = new float[size];
int i = 0;
if (key.equals(FFMPRecord.ALL)) {
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(entry.getValue().getLookupId());
dataRec[i] = bd.getValue();
i++;
}
}
} else {
for (Long pfaf : map.keySet()) {
ArrayList<Long> vgbpfafs = template
.getVGBsInAggregate(pfaf,
record.getSiteKey(), key);
if (vgbpfafs != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(pfaf);
dataRec[i] = bd.getValue();
i++;
}
}
}
// NAME | GROUP | array |Dimension | dimensions
IDataRecord rec = new FloatDataRecord(key,
record.getDataURI() + "/" + domain.getCwa(),
dataRec, 1, new long[] { size });
dataStore.addDataRecord(rec);
}
float[] dataRec = new float[size];
int i = 0;
for (Entry<String, FFMPVirtualGageBasinMetaData> entry : vmap
.entrySet()) {
if (entry.getValue() != null) {
FFMPVirtualGageBasin bd = (FFMPVirtualGageBasin) fbd
.get(entry.getValue().getLookupId());
dataRec[i] = bd.getValue();
i++;
}
}
// NAME | GROUP | array |Dimension | dimensions
IDataRecord rec = new FloatDataRecord(FFMPRecord.ALL,
record.getDataURI() + "/" + domain.getCwa(),
dataRec, 1, new long[] { size });
dataStore.addDataRecord(rec);
} else {
statusHandler.handle(Priority.DEBUG, "No VGB's in domain: "
+ domain.getCwa());
@ -170,44 +144,40 @@ public class FFMPDao extends PluginDao {
else {
for (String huc : record.getBasinsMap().keySet()) {
if (record.getBasinData() != null) {
if (record.getBasinData(huc) != null) {
for (DomainXML domain : template.getDomains()) {
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(),
FFMPRecord.ALL);
FFMPBasinData fbd = record.getBasinData();
// ignore data outside domain
if (map.size() > 0 && fbd.getBasins().size() > 0) {
int size = map.size();
LinkedHashMap<Long, ?> map = template.getMap(
record.getSiteKey(), domain.getCwa(), huc);
FFMPBasinData fbd = record.getBasinData(huc);
// ignore data outside domain
if (map.size() > 0 && fbd.getBasins().size() > 0) {
int size = map.size();
float[] dataRec = new float[size];
int i = 0;
// write individual basins, use template, preserves
// ordering
for (Long pfaf : map.keySet()) {
FFMPBasin bd = fbd.get(pfaf);
if (bd != null) {
dataRec[i] = bd.getValue();
i++;
}
float[] dataRec = new float[size];
int i = 0;
// write individual basins, use template, preserves
// ordering
for (Long pfaf : map.keySet()) {
FFMPBasin bd = fbd.get(pfaf);
if (bd != null) {
dataRec[i] = bd.getValue();
i++;
}
// NAME | GROUP | array |Dimension | dimensions
if (i > 0) {
IDataRecord rec = new FloatDataRecord(huc,
record.getDataURI() + "/"
+ domain.getCwa(), dataRec, 1,
new long[] { size });
dataStore.addDataRecord(rec);
}
} else {
statusHandler.handle(
Priority.DEBUG,
"Data outside of domain: "
+ domain.getCwa());
}
// NAME | GROUP | array |Dimension | dimensions
if (i > 0) {
IDataRecord rec = new FloatDataRecord(
FFMPRecord.ALL, record.getDataURI() + "/"
+ domain.getCwa(), dataRec, 1,
new long[] { size });
dataStore.addDataRecord(rec);
}
} else {
statusHandler.handle(Priority.DEBUG,
"Data outside of domain: " + domain.getCwa());
}
}
}

6
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/dataaccess/FFMPGeometryFactory.java
View file

@ -64,6 +64,7 @@ import com.vividsolutions.jts.geom.Geometry;
* ------------ ---------- ----------- --------------------------
* Jan 24, 2013 1552 mpduff Initial creation
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* </pre>
*
@ -126,8 +127,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory {
for (Map.Entry<String, Object> es : map.entrySet()) {
FFMPRecord rec = (FFMPRecord) es.getValue();
try {
rec.retrieveMapFromDataStore(templates, (String) request
.getIdentifiers().get(HUC));
rec.retrieveMapFromDataStore(templates);
} catch (Exception e) {
throw new DataRetrievalException(
"Failed to retrieve the IDataRecord for PluginDataObject: "
@ -204,7 +204,7 @@ public class FFMPGeometryFactory extends AbstractDataPluginFactory {
String siteKey = (String) request.getIdentifiers().get(SITE_KEY);
String cwa = (String) request.getIdentifiers().get(WFO);
FFMPBasinData basinData = rec.getBasinData(huc);
FFMPBasinData basinData = rec.getBasinData();
Map<Long, FFMPBasin> basinDataMap = basinData.getBasins();

521
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -32,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.regex.Pattern;
import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage;
import com.raytheon.uf.common.dataplugin.message.DataURINotificationMessage;
import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
import com.raytheon.edex.urifilter.URIFilter;
import com.raytheon.edex.urifilter.URIGenerateMessage;
@ -123,6 +123,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
* 02/25/13 1660 D. Hladky Redesigned data flow for FFTI in order to have only one mosaic piece in memory at a time.
* 03/13/13 1478 D. Hladky non-FFTI mosaic containers weren't getting ejected. Made it so that they are ejected after processing as well.
* 03/22/13 1803 D. Hladky Fixed broken performance logging for ffmp.
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -131,9 +132,10 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
public class FFMPGenerator extends CompositeProductGenerator implements
MonitorConfigListener {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(FFMPGenerator.class);
/**
* Public constructor for FFMPGenerator
*
@ -152,13 +154,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private static final String productType = "ffmp";
/**
* The thought was this will eventually be dynamic when front end can support it.
* The thought was this will eventually be dynamic when front end can
* support it.
*/
public static final int SOURCE_CACHE_TIME = 6;
/**
* The thought was this will eventually be dynamic, static in AWIPS I.
* This is the time back limit for Flash Flood Guidance sources
* The thought was this will eventually be dynamic, static in AWIPS I. This
* is the time back limit for Flash Flood Guidance sources
*/
public static final int FFG_SOURCE_CACHE_TIME = 24;
@ -388,7 +391,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.PROBLEM,
"Couldn't create FFMP Filter.." + " primary Domain: "
+ domain.getCwa()
+ " this RUNNER is not a viable FFMP config.", e);
+ " this RUNNER is not a viable FFMP config.",
e);
}
}
@ -440,7 +444,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
@Override
public void generateProduct(URIGenerateMessage genMessage) {
if (loaded) {
if (loaded) {
try {
long time = System.currentTimeMillis();
this.config = new FFMPConfig(
@ -477,7 +481,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
} catch (InterruptedException e) {
statusHandler.handle(Priority.ERROR, "Process thread had been interupted!", e);
statusHandler.handle(Priority.ERROR,
"Process thread had been interupted!", e);
}
}
@ -554,6 +559,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private class ProcessProduct implements Runnable {
SourceXML ffmpProduct;
FFMPGenerator generator;
@Override
@ -592,7 +598,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
HashMap<String, Object> dataHash = config.getSourceData(ffmpProduct
.getSourceName());
FFMPRunXML runner = getRunConfig().getRunner(config.getCWA());
// process all of the dataKeys for this source
@ -684,19 +690,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements
ffmpRec, template);
ffmpRec = ffmp.processFFMP(ffmpProduct);
ffmpRec.constructDataURI();
if (ffmpRec != null) {
persistRecord(ffmpRec);
processDataContainer(ffmpRec, siteKey);
// Now that we have the data container,
// Now that we have the data container,
// we can process FFTI for this piece of the mosaic
if (ffmp.isFFTI()) {
fftiDone = false;
FFTISourceXML fftiSource = ffmp.getFFTISource();
// This only runs once for the site key loop
if (!fftiSources.contains(fftiSource)) {
FFTIProcessor ffti = new FFTIProcessor(
@ -735,16 +741,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
}
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
String sourceSiteDataKey = getSourceSiteDataKey(source,
dataKey, ffmpRec);
String sourceSiteDataKey = getSourceSiteDataKey(
source, dataKey, ffmpRec);
ffmpData.remove(sourceSiteDataKey);
statusHandler.info("Removing from memory: "+sourceSiteDataKey);
statusHandler.info("Removing from memory: "
+ sourceSiteDataKey);
}
}
}
@ -970,8 +977,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteSourceFileName(sourceId));
try {
sbl = SerializationUtil
.transformFromThrift(SourceBinList.class, FileUtil.file2bytes(f.getFile(), true));
sbl = SerializationUtil.transformFromThrift(SourceBinList.class,
FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName());
@ -979,8 +986,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR,
"Unable to read file " + f.getName());
} catch (IOException ioe) {
statusHandler.handle(Priority.ERROR, "General IO problem with file "
+ f.getName(), ioe);
statusHandler.handle(Priority.ERROR,
"General IO problem with file " + f.getName(), ioe);
}
return sbl;
@ -992,8 +999,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public String getAbsoluteSourceFileName(String sourceId) {
return productType + File.separator + "sources" + File.separator + sourceId
+ ".bin";
return productType + File.separator + "sources" + File.separator
+ sourceId + ".bin";
}
/**
@ -1098,7 +1105,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
*/
public FFMPDataContainer getFFMPDataContainer(String siteSourceKey,
ArrayList<String> hucs, Date backDate) {
Date backDate) {
FFMPDataContainer container = ffmpData.get(siteSourceKey);
@ -1112,7 +1119,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
siteKey = parts[0];
}
container = loadFFMPDataContainer(siteSourceKey, hucs, siteKey,
container = loadFFMPDataContainer(siteSourceKey, siteKey,
config.getCWA(), backDate);
if (container != null) {
@ -1194,180 +1201,157 @@ public class FFMPGenerator extends CompositeProductGenerator implements
/**
* Process the ffmp data container
*
* @param ffmpRec
* @param productKey
*/
public void processDataContainer(FFMPRecord ffmpRec, String productKey) {
public void processDataContainer(FFMPRecord ffmpRec, String productKey) {
String sourceName = null;
Date backDate = null;
String sourceSiteDataKey = null;
FFMPDataContainer fdc = null;
boolean write = true;
String sourceName = null;
Date backDate = null;
String sourceSiteDataKey = null;
FFMPDataContainer fdc = null;
boolean write = true;
try {
// write out the fast loader cache file
long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
String dataKey = ffmpRec.getDataKey();
try {
// write out the fast loader cache file
long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
String dataKey = ffmpRec.getDataKey();
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
// deal with setting of needed HUCS
ArrayList<String> hucs = template.getTemplateMgr().getHucLevels();
// pull from disk if there
fdc = getFFMPDataContainer(sourceSiteDataKey, backDate);
if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType())
|| source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
hucs.clear();
hucs.add(FFMPRecord.ALL);
} else {
hucs.remove(FFMPRecord.VIRTUAL);
}
// brand new or initial load up
if (fdc == null || !loadedData.contains(sourceSiteDataKey)) {
// pull from disk if there
fdc = getFFMPDataContainer(sourceSiteDataKey, hucs, backDate);
long time = System.currentTimeMillis();
fdc = new FFMPDataContainer(sourceSiteDataKey);
fdc = FFTIProcessor.populateDataContainer(fdc, template,
backDate, ffmpRec.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec.getSiteKey());
// brand new or initial load up
if (fdc == null || !loadedData.contains(sourceSiteDataKey)) {
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Populated new source: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
long time = System.currentTimeMillis();
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs);
fdc = FFTIProcessor.populateDataContainer(fdc, template, hucs,
backDate, ffmpRec.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec.getSiteKey());
} else {
if (source.getSourceType().equals(
SOURCE_TYPE.GAGE.getSourceType())
|| source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
hucs.clear();
hucs.add(FFMPRecord.ALL);
} else {
hucs.remove(FFMPRecord.VIRTUAL);
}
long time = System.currentTimeMillis();
// guidance sources are treated as a mosaic and are handled
// differently. They are force read at startup.
// This is the main line sequence a source will take when
// updated.
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Populated new source: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
Date newDate = fdc.getNewest();
Date oldDate = fdc.getOldest();
} else {
if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in
// existing source container, this will fill in
// gaps
// if
// they exist.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, newDate, ffmpRec.getDataTime()
.getRefTime(), ffmpRec.getWfo(),
source, ffmpRec.getSiteKey());
long time = System.currentTimeMillis();
// guidance sources are treated as a mosaic and are handled
// differently. They are force read at startup.
// This is the main line sequence a source will take when
// updated.
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
} else if (oldDate
.after(new Date(
backDate.getTime()
- (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for
// existing source container, this happens if
// the
// ingest was turned off for some period of
// time.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, backDate, oldDate,
ffmpRec.getWfo(), source,
ffmpRec.getSiteKey());
}
}
Date newDate = fdc.getNewest();
Date oldDate = fdc.getOldest();
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Checked Source files: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
}
}
if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in
// existing source container, this will fill in
// gaps
// if
// they exist.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, null, newDate, ffmpRec
.getDataTime().getRefTime(),
ffmpRec.getWfo(), source, ffmpRec
.getSiteKey());
// add current record data
} else if (oldDate
.after(new Date(
backDate.getTime()
- (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for
// existing source container, this happens if
// the
// ingest was turned off for some period of
// time.
fdc = FFTIProcessor.populateDataContainer(fdc,
template, null, backDate, oldDate,
ffmpRec.getWfo(), source,
ffmpRec.getSiteKey());
}
}
fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source,
ffmpRec.getBasinData(), ffmpRec.getSiteKey());
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG,
"Checked Source files: in " + (time2 - time)
+ " ms: source: " + sourceSiteDataKey);
}
}
// cache it temporarily for FFTI use
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
// only write last one
write = false;
// add current record data
for (String huc : hucs) {
fdc.addFFMPEntry(ffmpRec.getDataTime().getRefTime(), source,
ffmpRec.getBasinData(huc), huc, ffmpRec.getSiteKey());
}
if (!ffmpData.containsKey(sourceSiteDataKey)) {
ffmpData.put(sourceSiteDataKey, fdc);
} else {
ffmpData.replace(sourceSiteDataKey, fdc);
}
}
// cache it temporarily for FFTI use
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
// only write last one
write = false;
statusHandler.handle(
Priority.INFO,
"Processed FFMPDataContainer: in "
+ (System.currentTimeMillis() - ptime)
+ " ms: source: " + sourceSiteDataKey);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"Failed Processing FFMPDataContainer" + e.getMessage());
if (!ffmpData.containsKey(sourceSiteDataKey)) {
ffmpData.put(sourceSiteDataKey, fdc);
} else {
ffmpData.replace(sourceSiteDataKey, fdc);
}
}
} finally {
// purge it up
if (fdc != null) {
// this is defensive for if errors get thrown
if (backDate == null) {
backDate = new Date((System.currentTimeMillis())
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
statusHandler.handle(
Priority.INFO,
"Processed FFMPDataContainer: in "
+ (System.currentTimeMillis() - ptime)
+ " ms: source: " + sourceSiteDataKey);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"Failed Processing FFMPDataContainer" + e.getMessage());
if (!fdc.isPurged()) {
fdc.purge(backDate);
}
} finally {
// purge it up
if (fdc != null) {
// this is defensive for if errors get thrown
if (backDate == null) {
backDate = new Date((System.currentTimeMillis())
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
if (!fdc.isPurged()) {
fdc.purge(backDate);
}
if (write) {
// write it out
writeAggregateRecord(fdc, sourceSiteDataKey);
}
}
}
}
if (write) {
// write it out
writeAggregateRecord(fdc, sourceSiteDataKey);
}
}
}
}
/**
* load existing container
@ -1379,7 +1363,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public FFMPDataContainer loadFFMPDataContainer(String sourceSiteDataKey,
ArrayList<String> hucs, String siteKey, String wfo, Date backDate) {
String siteKey, String wfo, Date backDate) {
FFMPDataContainer fdc = null;
FFMPAggregateRecord record = null;
@ -1396,7 +1380,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// condition for first time read in
if (fdc == null && record != null) {
// creates a place holder for this source
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs, record);
fdc = new FFMPDataContainer(sourceSiteDataKey, record);
populated = true;
}
@ -1404,9 +1388,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (record != null && !populated) {
fdc.setAggregateData(record);
}
// sometimes a record will sit around for a long time and it will have data going back to the last precip event
// this can be an enormous amount of time. Want to get the data dumped from memory ASAP.
// sometimes a record will sit around for a long time and it will have
// data going back to the last precip event
// this can be an enormous amount of time. Want to get the data dumped
// from memory ASAP.
if (fdc != null) {
fdc.purge(backDate);
}
@ -1443,9 +1429,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
*
* @param fdc
*/
public void writeAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) {
public void writeAggregateRecord(FFMPDataContainer fdc,
String sourceSiteDataKey) {
WriteAggregateRecord writer = new WriteAggregateRecord(fdc, sourceSiteDataKey);
WriteAggregateRecord writer = new WriteAggregateRecord(fdc,
sourceSiteDataKey);
writer.run();
}
@ -1458,7 +1446,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private class WriteAggregateRecord implements Runnable {
private FFMPDataContainer fdc;
private String sourceSiteDataKey;
public void run() {
@ -1470,10 +1458,12 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
public WriteAggregateRecord(FFMPDataContainer fdc, String sourceSiteDataKey) {
public WriteAggregateRecord(FFMPDataContainer fdc,
String sourceSiteDataKey) {
this.fdc = fdc;
this.sourceSiteDataKey = sourceSiteDataKey;
statusHandler.handle(Priority.DEBUG, "Created Aggregate Record Writer");
statusHandler.handle(Priority.DEBUG,
"Created Aggregate Record Writer");
}
/**
@ -1492,34 +1482,37 @@ public class FFMPGenerator extends CompositeProductGenerator implements
aggRecord.setWfo(config.getCWA());
// times for Guidance basins will be null
aggRecord.setTimes(fdc.getOrderedTimes());
for (FFMPBasinData fbd : fdc.getBasinMap().values()) {
fbd.serialize();
aggRecord.addBasinData(fbd);
}
fdc.getBasinData().serialize();
aggRecord.setBasins(fdc.getBasinData());
}
if (aggRecord.getBasinsMap().size() > 0) {
if (aggRecord.getBasins() != null) {
try {
StorageProperties sp = null;
String compression = PluginRegistry.getInstance()
.getRegisteredObject(productType).getCompression();
.getRegisteredObject(productType)
.getCompression();
if (compression != null) {
sp = new StorageProperties();
sp.setCompression(Compression.valueOf(compression));
}
byte[] bytes = SerializationUtil.transformToThrift(aggRecord);
byte[] bytes = SerializationUtil
.transformToThrift(aggRecord);
// NAME | GROUP | array |Dimension | size
IDataRecord rec = new ByteDataRecord(sourceSiteDataKey, config.getCWA(),
bytes, 1, new long[] { bytes.length });
File hdf5File = FFMPUtils.getHdf5File(config.getCWA(), sourceSiteDataKey);
IDataStore dataStore = DataStoreFactory.getDataStore(hdf5File);
// write it, allowing, and in fact encouraging replacing the last one
IDataRecord rec = new ByteDataRecord(sourceSiteDataKey,
config.getCWA(), bytes, 1,
new long[] { bytes.length });
File hdf5File = FFMPUtils.getHdf5File(config.getCWA(),
sourceSiteDataKey);
IDataStore dataStore = DataStoreFactory
.getDataStore(hdf5File);
// write it, allowing, and in fact encouraging replacing
// the last one
dataStore.addDataRecord(rec, sp);
dataStore.store(StoreOp.OVERWRITE);
@ -1538,7 +1531,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
@Override
public synchronized void configChanged(MonitorConfigEvent fce) {
@ -1703,8 +1695,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @return
*/
public String getAbsoluteFFTIFileName(String fftiName) {
return productType + File.separator + "ffti" + File.separator + fftiName
+ ".bin";
return productType + File.separator + "ffti" + File.separator
+ fftiName + ".bin";
}
/**
@ -1734,7 +1726,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
public ConcurrentHashMap<String, FFTIData> getFFTIDataContainer() {
return fftiData;
}
/**
* Get value for an individual piece of the puzzle
*
@ -1748,8 +1740,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
public FFTIAccum getAccumulationForSite(String fftiSourceKey,
String fftiSiteKey, String fftiDataKey, double duration, String unit) {
SourceXML ffmpSource = getSourceConfig()
.getSourceByDisplayName(fftiSourceKey);
SourceXML ffmpSource = getSourceConfig().getSourceByDisplayName(
fftiSourceKey);
FFTIAccum accumulator = null;
String siteDataKey = ffmpSource.getDisplayName() + "-" + fftiSiteKey
+ "-" + fftiDataKey;
@ -1775,24 +1767,23 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long cur = config.getDate().getTime();
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
FFMPDataContainer fdc = null;
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
fdc = getFFMPDataContainer(siteDataKey, hucs, backDate);
fdc = getFFMPDataContainer(siteDataKey, backDate);
if (fdc != null) {
FFMPBasinData fbd = fdc.getBasinData(FFMPRecord.ALL);
FFMPBasinData fbd = fdc.getBasinData();
// go over the list of CWAs gathering the pfaf list
ArrayList<Long> pfafs = new ArrayList<Long>();
ArrayList<String> cwaList = config.fdm.getCwaList();
Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(), duration, fftiSiteKey);
Double gap = FFTI.getGap(fdc, ffmpSource, config.getDate(),
duration, fftiSiteKey);
if (!Double.isNaN(gap)) {
for (Long key : fbd.getBasins().keySet()) {
@ -1831,14 +1822,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
ffmpData.remove(siteDataKey);
statusHandler.info("Removing from memory: "+siteDataKey);
statusHandler.info("Removing from memory: " + siteDataKey);
accumulator.setReset(false);
writeFFTIData(siteDataKey, accumulator);
}
return accumulator;
}
/**
* Gets the ratio and difference values for this site
*
@ -1880,24 +1871,24 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long cur = config.getDate().getTime();
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
// make sure we have data
Date ffgBackDate = new Date(config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
Date ffgBackDate = new Date(
config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
String primarySource = fscm.getPrimarySource(ffmpQSource);
ProductXML product = fscm.getProduct(primarySource);
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
FFMPDataContainer guidContainer = getFFMPDataContainer(
ffgType, hucs, ffgBackDate);
FFMPDataContainer guidContainer = getFFMPDataContainer(ffgType,
ffgBackDate);
long guidSourceExpiration = 0l;
if (guidContainer == null) {
guidContainer = new FFMPDataContainer(ffgType, hucs);
guidContainer = new FFMPDataContainer(ffgType);
}
for (SourceXML iguidSource : product
@ -1905,7 +1896,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (guidSourceExpiration == 0l) {
guidSourceExpiration = iguidSource
.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
.getExpirationMinutes(qSiteKey)
* TimeUtil.MILLIS_PER_MINUTE;
break;
}
}
@ -1918,15 +1910,17 @@ public class FFMPGenerator extends CompositeProductGenerator implements
+ " " + qSourceKey + " " + " comparison.");
return values;
}
String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-" + qSiteKey + "-"+ qSiteKey;
FFMPDataContainer qpeContainer = getFFMPDataContainer(qpeSiteSourceDataKey, hucs, backDate);
String qpeSiteSourceDataKey = ffmpQSource.getSourceName() + "-"
+ qSiteKey + "-" + qSiteKey;
FFMPDataContainer qpeContainer = getFFMPDataContainer(
qpeSiteSourceDataKey, backDate);
if (qpeContainer != null) {
// go over the list of CWAs gathering the pfaf list
ArrayList<Long> pfafs = new ArrayList<Long>();
ArrayList<String> cwaList = config.fdm.getCwaList();
FFMPBasinData fbd = qpeContainer.getBasinData(FFMPRecord.ALL);
FFMPBasinData fbd = qpeContainer.getBasinData();
for (Long key : fbd.getBasins().keySet()) {
for (String cwa : cwaList) {
@ -1953,22 +1947,22 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
Double gap = FFTI.getGap(qpeContainer, ffmpQSource, config.getDate(), duration,
qSiteKey);
Double gap = FFTI.getGap(qpeContainer, ffmpQSource,
config.getDate(), duration, qSiteKey);
if (!Double.isNaN(gap)) {
List<Float> qpes = qpeContainer.getBasinData(FFMPRecord.ALL)
List<Float> qpes = qpeContainer.getBasinData()
.getAccumValues(pfafs, backDate, config.getDate(),
expirationTime, false);
FFMPGuidanceInterpolation interpolator = new FFMPGuidanceInterpolation(
fscm, product, frcm.getRunner(
config.getCWA()).getProduct(qSiteKey),
primarySource, ffgType, qSiteKey);
fscm, product, frcm.getRunner(config.getCWA())
.getProduct(qSiteKey), primarySource,
ffgType, qSiteKey);
interpolator.setInterpolationSources(duration);
List<Float> guids = guidContainer.getBasinData(FFMPRecord.ALL)
List<Float> guids = guidContainer.getBasinData()
.getGuidanceValues(pfafs, interpolator,
guidSourceExpiration);
@ -1982,44 +1976,41 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// replace or insert it
ffmpData.remove(qpeSiteSourceDataKey);
statusHandler.info("Removing from memory: "+qpeSiteSourceDataKey);
statusHandler.info("Removing from memory: " + qpeSiteSourceDataKey);
values.setReset(false);
writeFFTIData(siteDataKey, values);
}
return values;
}
/**
* Persist the record that has finished processing.
* This is different than other DAT tools.
* Other tools wait until all are finished processing
* before persisting. FFMP persists as it goes in order
* to lessen the data surge being sent to pypies.
* Persist the record that has finished processing. This is different than
* other DAT tools. Other tools wait until all are finished processing
* before persisting. FFMP persists as it goes in order to lessen the data
* surge being sent to pypies.
*
* @param record
* @return
*/
private synchronized void persistRecord(FFMPRecord record) {
// persist out this record
try {
setPluginDataObjects(new FFMPRecord[]{record});
setPluginDao(new FFMPDao(getCompositeProductType(),
template, fscm, config.getCWA()));
setPluginDataObjects(new FFMPRecord[] { record });
setPluginDao(new FFMPDao(getCompositeProductType(), template, fscm,
config.getCWA()));
persistRecords();
fireTopicUpdate();
// clear out pdos that are written
pdos = null;
} catch (PluginException e) {
statusHandler.handle(Priority.PROBLEM, "Couldn't persist the record.", e);
statusHandler.handle(Priority.PROBLEM,
"Couldn't persist the record.", e);
}
}
/**
* Find siteSourceDataKey
*
@ -2028,25 +2019,25 @@ public class FFMPGenerator extends CompositeProductGenerator implements
* @param ffmpRec
* @return
*/
private String getSourceSiteDataKey(SourceXML source, String dataKey, FFMPRecord ffmpRec) {
private String getSourceSiteDataKey(SourceXML source, String dataKey,
FFMPRecord ffmpRec) {
String sourceName = source.getSourceName();
String sourceSiteDataKey = null;
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
if (source.getSourceType().equals(SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() + "-"
+ dataKey;
}
return sourceSiteDataKey;
}
/**
* Log process statistics
*
@ -2083,4 +2074,4 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
}
}

21
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java
View file

@ -20,7 +20,6 @@ package com.raytheon.uf.edex.plugin.ffmp.common;
* further licensing information.
**/
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
@ -45,6 +44,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* 29 July, 2012 578 dhladky memory work
* 27 Jan, 2013 1478 dhladky Changed arraylist to list for times, more constants
* 02/01/13 1569 D. Hladky Added constants
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -93,15 +93,14 @@ public class FFMPInterpolatedGuidanceDelay {
public boolean calculateDelayedGuidance() {
boolean delayGuidance = false;
ArrayList<String> hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
FFMPDataContainer qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName()
+ "-" + siteKey + "-" + siteKey, hucs, backDate);
FFMPDataContainer qpeContainer = generator.getFFMPDataContainer(
qpeSource.getSourceName() + "-" + siteKey + "-" + siteKey,
backDate);
// Don't do anything, we have no QPE
if (qpeContainer != null) {
long expirationTime = qpeSource.getExpirationMinutes(siteKey)
* TimeUtil.MILLIS_PER_MINUTE;
// determine lag_time
@ -115,8 +114,8 @@ public class FFMPInterpolatedGuidanceDelay {
.getOrderedTimes(currentRecord.getDataTime().getRefTime());
// EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld)
for (Entry<Long, FFMPBasin> entry : currentRecord.getBasinsMap()
.get(FFMPRecord.ALL).getBasins().entrySet()) {
for (Entry<Long, FFMPBasin> entry : currentRecord.getBasinData()
.getBasins().entrySet()) {
FFMPBasin currBasin = entry.getValue();
FFMPGuidanceBasin oldBasin = (FFMPGuidanceBasin) previousGuidanceData
.get(entry.getKey());
@ -136,8 +135,8 @@ public class FFMPInterpolatedGuidanceDelay {
// this is essentially a ratio of the first accumulation
// step increment
// to the total amount over this time window.
FFMPBasin qpeBasin = qpeContainer.getBasinData(
FFMPRecord.ALL).get(entry.getKey());
FFMPBasin qpeBasin = qpeContainer.getBasinData().get(
entry.getKey());
if (qpeBasin != null) {
float intervalAccum = qpeBasin.getAccumValue(
@ -160,7 +159,7 @@ public class FFMPInterpolatedGuidanceDelay {
currBasin.setValue(newDate, val);
}
}
delayGuidance = true;
}

107
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPProcessor.java
View file

@ -98,6 +98,7 @@ import com.vividsolutions.jts.geom.Polygon;
* 02/01/13 1569 D. Hladky Added constants
* 02/25/13 1660 D. Hladky FFTI design change to help mosaic processing.
* 05/01/2013 15684 zhao Unlock when Exception caught
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
* </pre>
*
* @author dhladky
@ -214,29 +215,6 @@ public class FFMPProcessor {
processSource();
}
// don't do gages for anything other than "ALL"
if ((sourceId != null)
&& !source.getSourceType().equals(
FFMPSourceConfigurationManager.SOURCE_TYPE.GAGE
.getSourceType())) {
//ArrayList<String> hucs = template.getTemplateMgr().getHucLevels();// DR 15514
String[] hucs = template.getTemplateMgr().getHucLevelsInArray();// DR 15514
synchronized (hucs) {
if (hucs != null) {
//for (String huc : hucs) {
for(int i=0; i<hucs.length; i++){
String huc = hucs[i];
if (huc != null) {
if (!huc.equals("ALL") || !huc.equals("VIRTUAL")) {
setValues(huc);
}
}
}
}
}
}
statusHandler.handle(Priority.INFO,
"Processed Source: " + ffmpRec.getSourceName() + " sitekey: "
+ siteKey + " dataKey: " + dataKey + " time: "
@ -471,7 +449,7 @@ public class FFMPProcessor {
if (sourceId != null) {
for (Long key : map.keySet()) {
FFMPBasin basin = getBasin(key, FFMPRecord.ALL);
FFMPBasin basin = getBasin(key);
Date date = null;
Float val = null;
@ -595,7 +573,7 @@ public class FFMPProcessor {
for (Long key : map.keySet()) {
FFMPBasin basin = getBasin(key, "ALL");
FFMPBasin basin = getBasin(key);
float val = 0.0f;
val = processGrib(key, domain.getCwa());
setBasin(basin, val);
@ -643,11 +621,9 @@ public class FFMPProcessor {
Date backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()-(FFMPGenerator.SOURCE_CACHE_TIME * TimeUtil.MILLIS_PER_HOUR));
ArrayList<String> hucs = new ArrayList<String>();
hucs.add("ALL");
FFMPDataContainer ffgContainer = generator
.getFFMPDataContainer(sourceNameString, hucs,
.getFFMPDataContainer(sourceNameString,
backDate);
if (ffgContainer != null
@ -688,7 +664,7 @@ public class FFMPProcessor {
siteKey, guidFrequency, source,
qpeSource, previousDate, recdate,
generator,
ffgContainer.getBasinData(FFMPRecord.ALL),
ffgContainer.getBasinData(),
ffmpRec);
boolean delayGuidance = figd
@ -811,9 +787,9 @@ public class FFMPProcessor {
*
* @param ffmp
*/
private FFMPBasinData getBasinData(String huc) {
private FFMPBasinData getBasinData() {
return ffmpRec.getBasinData(huc);
return ffmpRec.getBasinData();
}
/**
@ -823,15 +799,11 @@ public class FFMPProcessor {
* @param huc
* @return
*/
private FFMPBasin getBasin(Long pfaf, String huc) {
FFMPBasin basin = getBasinData(huc).get(pfaf);
private FFMPBasin getBasin(Long pfaf) {
FFMPBasin basin = getBasinData().get(pfaf);
if (basin == null) {
if (huc.equals(FFMPRecord.ALL)) {
basin = new FFMPBasin(pfaf, false);
} else {
basin = new FFMPBasin(pfaf, true);
}
getBasinData(huc).put(pfaf, basin);
basin = new FFMPBasin(pfaf, false);
getBasinData().put(pfaf, basin);
}
return basin;
}
@ -845,64 +817,15 @@ public class FFMPProcessor {
*/
private FFMPVirtualGageBasin getVirtualBasin(String lid, Long pfaf,
String huc) {
FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData(huc)
FFMPVirtualGageBasin basin = (FFMPVirtualGageBasin) getBasinData()
.get(pfaf);
if (basin == null) {
if (huc.equals(FFMPRecord.ALL)) {
basin = new FFMPVirtualGageBasin(lid, pfaf, false);
} else {
basin = new FFMPVirtualGageBasin(lid, pfaf, true);
}
getBasinData(huc).put(pfaf, basin);
basin = new FFMPVirtualGageBasin(lid, pfaf, false);
getBasinData().put(pfaf, basin);
}
return basin;
}
/**
* Sets the values for the aggregated basins
*
* @param type
* @return
*/
private void setValues(String huc) {
try {
// Get basins for level, we process VGB's differently because it is
// a
// special case
if (!huc.equals(FFMPRecord.VIRTUAL) && !huc.equals(FFMPRecord.ALL)) {
for (DomainXML domain : template.getDomains()) {
LinkedHashMap<Long, ?> map = template.getMap(siteKey,
domain.getCwa(), huc);
for (Long pfaf : map.keySet()) {
FFMPBasin basin = getBasin(pfaf, huc);
Float val = 0.0f;
// average values
try {
ArrayList<Long> aggPfafs = template
.getAggregatePfafs(pfaf, siteKey, huc);
ArrayList<Double> areas = template
.getAreas(aggPfafs);
val = ffmpRec.getBasinData(FFMPRecord.ALL).getAverageValue(
aggPfafs, areas);
} catch (Exception e) {
// Value is NAN, ignore it.
}
basin.setValue(config.getDate(), val);
}
}
}
} catch (Exception e) {
config.getGenerator().logger.error("Unable to process " + huc
+ " level data");
}
}
/**
* Process the PDO for a geometry
*

30
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTIProcessor.java
View file

@ -57,6 +57,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* </pre>
* Apr 16, 2013 1912 bsteffen Initial bulk hdf5 access for ffmp
* Apr 18, 2013 1919 dhladky Fixed VGB breakage
* Jul 15, 2013 2184 dhladky Remove all HUC's for storage except ALL
*
* @author dhladky
* @version 1.0
@ -182,8 +183,8 @@ public class FFTIProcessor {
*/
public static FFMPDataContainer populateDataContainer(
FFMPDataContainer sourceContainer, FFMPTemplates template,
ArrayList<String> hucs, Date startDate, Date endDate, String wfo,
SourceXML source, String siteKey) {
Date startDate, Date endDate, String wfo, SourceXML source,
String siteKey) {
ArrayList<String> uris = getUris(startDate, endDate, wfo, source,
siteKey);
@ -209,19 +210,11 @@ public class FFTIProcessor {
if (!contains) {
try {
if (hucs == null) {
hucs = new ArrayList<String>();
hucs.add(FFMPRecord.ALL);
}
for (String huc : hucs) {
rec = populateRecord(rec, huc, template);
FFMPBasinData newData = rec.getBasinData(huc);
sourceContainer.addFFMPEntry(rec.getDataTime()
.getRefTime(), source, newData, huc, siteKey);
}
rec = populateRecord(rec, template);
FFMPBasinData newData = rec.getBasinData();
sourceContainer.addFFMPEntry(
rec.getDataTime().getRefTime(), source, newData,
siteKey);
// System.out.println("Adding Time: "
// + rec.getDataTime().getRefTime());
@ -309,8 +302,7 @@ public class FFTIProcessor {
* @return
* @throws PluginException
*/
public static FFMPRecord populateRecord(FFMPRecord rec, String huc,
FFMPTemplates template) throws PluginException {
public static FFMPRecord populateRecord(FFMPRecord rec, FFMPTemplates template) throws PluginException {
try {
@ -328,9 +320,9 @@ public class FFTIProcessor {
// check for gage(VGB) types, if so process as a VGB
if (source.getSourceType().equals(SOURCE_TYPE.GAGE.getSourceType())) {
rec.retrieveVirtualMapFromDataStore(template, huc);
rec.retrieveVirtualMapFromDataStore(template);
} else {
rec.retrieveMapFromDataStore(template, huc);
rec.retrieveMapFromDataStore(template);
}
// System.out.println("Size of huc: "