xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1478 FFMP cache file read/writes overloading NAS, more efficiency

Browse source 
Change-Id: Ib5f49d9c1551b72a3620629437386d52c12b0ecf

Former-commit-id: 7b49430e34 [formerly c0712160d7] [formerly 7b49430e34 [formerly c0712160d7] [formerly 7af5aab08a [formerly 9c7d813ee094864e0a7da60d54dee5424c50efba]]]
Former-commit-id: 7af5aab08a
Former-commit-id: b134741e4c [formerly f6bd4ea0fa]
Former-commit-id: 69477ec246
This commit is contained in:
Dave Hladky 2013-02-05 11:52:59 -06:00
parent 491d2c3e63
commit d8ce48e79d
22 changed files with 900 additions and 506 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java
View file

@ -28,6 +28,7 @@ import org.eclipse.ui.PlatformUI;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin; import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData; import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPCacheRecord; import com.raytheon.uf.common.dataplugin.ffmp.FFMPCacheRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin; import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceInterpolation; import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceInterpolation;
@ -90,6 +91,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
* 04/03/10 4494 D. Hladky Initial release * 04/03/10 4494 D. Hladky Initial release
* 12/07/12 1353 rferrel Changes for non-blocking FFMPSplash. * 12/07/12 1353 rferrel Changes for non-blocking FFMPSplash.
* 01/10/13 1475 D. Hladky Cleaned up some logging. * 01/10/13 1475 D. Hladky Cleaned up some logging.
* 01/27/13 1478 D. Hladky revamped cache file format, removed duplicate times
* *
* </pre> * </pre>
* *
@ -475,28 +477,21 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey * @param siteKey
* @param dataKey * @param dataKey
* @param source * @param source
* @param huc
*/ */
public void insertFFMPData(FFMPBasinData data, String siteKey, public void insertFFMPData(FFMPAggregateRecord data, String siteKey,
String source, String huc) { String source) {
final String fsiteKey = siteKey; if (ffmpData.containsKey(siteKey)) {
final FFMPBasinData fdata = data; if (ffmpData.get(siteKey).containsKey(source)) {
final String fsource = source; for (Entry<String, FFMPBasinData> entry : data.getBasinsMap()
final String fhuc = huc; .entrySet()) {
FFMPBasinData basinData = entry.getValue();
VizApp.runAsync(new Runnable() { basinData.populate(data.getTimes());
@Override ffmpData.get(siteKey).get(source)
public void run() { .setCacheData(basinData, basinData.getHucLevel());
if (ffmpData.containsKey(fsiteKey)) {
if (ffmpData.get(fsiteKey).containsKey(fsource)) {
ffmpData.get(fsiteKey).get(fsource)
.setBasinBuddyData(fdata, fhuc);
}
} }
} }
}); }
} }
/** /**
@ -530,54 +525,35 @@ public class FFMPMonitor extends ResourceMonitor {
public void populateFFMPBasin(String dataUri, String siteKey, public void populateFFMPBasin(String dataUri, String siteKey,
String source, String phuc, FFMPBasin basin) throws VizException { String source, String phuc, FFMPBasin basin) throws VizException {
final String fdataUri = dataUri; if (dataUri != null) {
final String fsiteKey = siteKey; ConcurrentMap<String, String> uris = getUriMap(siteKey, source,
final String fhuc = phuc; phuc);
final String fsource = source; if (!uris.containsKey(dataUri)) {
final FFMPBasin fbasin = basin; try {
SourceXML sourceXML = fscm.getSource(source);
FFMPCacheRecord ffmpRec = populateFFMPRecord(true, dataUri,
siteKey, source, phuc);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory.getDataStore(loc);
VizApp.runAsync(new Runnable() { if (sourceXML.getSourceType().equals(
@Override SOURCE_TYPE.GAGE.getSourceType())
public void run() { && phuc.equals("ALL")) {
ffmpRec.retrieveVirtualBasinFromDataStore(dataStore,
if (fdataUri != null) { dataUri, getTemplates(siteKey), ffmpRec
ConcurrentMap<String, String> uris = getUriMap(fsiteKey, .getDataTime().getRefTime(), basin);
fsource, fhuc); } else {
if (!uris.containsKey(fdataUri)) { ffmpRec.retrieveBasinFromDataStore(dataStore, dataUri,
try { getTemplates(siteKey), phuc, ffmpRec
SourceXML sourceXML = fscm.getSource(fsource); .getDataTime().getRefTime(), ffmpRec
FFMPCacheRecord ffmpRec = populateFFMPRecord(true, .getSourceName(), basin);
fdataUri, fsiteKey, fsource, fhuc);
// FFMPRecord ffmpRec =
// loadRecordFromDatabase(fdataUri);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory
.getDataStore(loc);
if (sourceXML.getSourceType().equals(
SOURCE_TYPE.GAGE.getSourceType())
&& fhuc.equals("ALL")) {
ffmpRec.retrieveVirtualBasinFromDataStore(
dataStore, fdataUri,
getTemplates(fsiteKey), ffmpRec
.getDataTime().getRefTime(),
fbasin);
} else {
ffmpRec.retrieveBasinFromDataStore(dataStore,
fdataUri, getTemplates(fsiteKey), fhuc,
ffmpRec.getDataTime().getRefTime(),
ffmpRec.getSourceName(), fbasin);
}
} catch (Throwable e) {
statusHandler
.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, "
+ fdataUri, e);
}
} }
} catch (Throwable e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, " + dataUri, e);
} }
} }
}); }
} }
/** /**

335
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java
View file

@ -29,8 +29,9 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.NavigableMap; import java.util.NavigableMap;
import java.util.zip.GZIPInputStream;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData; import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager; import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.xml.FFMPRunXML; import com.raytheon.uf.common.monitor.xml.FFMPRunXML;
import com.raytheon.uf.common.monitor.xml.ProductRunXML; import com.raytheon.uf.common.monitor.xml.ProductRunXML;
@ -40,6 +41,10 @@ import com.raytheon.uf.common.ohd.AppsDefaults;
import com.raytheon.uf.common.serialization.DynamicSerializationManager; import com.raytheon.uf.common.serialization.DynamicSerializationManager;
import com.raytheon.uf.common.serialization.DynamicSerializationManager.SerializationType; import com.raytheon.uf.common.serialization.DynamicSerializationManager.SerializationType;
import com.raytheon.uf.common.serialization.SerializationException; import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.viz.core.VizApp; import com.raytheon.uf.viz.core.VizApp;
import com.raytheon.uf.viz.monitor.ffmp.FFMPMonitor; import com.raytheon.uf.viz.monitor.ffmp.FFMPMonitor;
import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FFMPConfig; import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FFMPConfig;
@ -57,6 +62,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent;
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 28 Feb, 2011 7587 dhladky Initial creation * 28 Feb, 2011 7587 dhladky Initial creation
* 25 Jan, 2012 DR13839 gzhang Handle Uris and Huc processing * 25 Jan, 2012 DR13839 gzhang Handle Uris and Huc processing
* 01/27/13 1478 D. Hladky revamped the cache file format to help NAS overloading
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -64,8 +70,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent;
*/ */
public class FFMPDataLoader extends Thread { public class FFMPDataLoader extends Thread {
// private static final transient IUFStatusHandler statusHandler = UFStatus private static final IUFStatusHandler statusHandler = UFStatus.getHandler(FFMPDataLoader.class);
// .getHandler(FFMPDataLoader.class);
private String sharePath = null; private String sharePath = null;
@ -154,7 +159,7 @@ public class FFMPDataLoader extends Thread {
ProductRunXML productRun = runner.getProduct(siteKey); ProductRunXML productRun = runner.getProduct(siteKey);
ArrayList<String> qpfSources = new ArrayList<String>(); ArrayList<String> qpfSources = new ArrayList<String>();
String layer = config.getFFMPConfigData().getLayer();
boolean isProductLoad = true; boolean isProductLoad = true;
String rateURI = null; String rateURI = null;
@ -225,179 +230,159 @@ public class FFMPDataLoader extends Thread {
hucsToLoad.add("ALL"); hucsToLoad.add("ALL");
} }
for (String phuc : hucsToLoad) { if (isDone) {
return;
}
if (isDone) { // rate
return; if (rateURI != null) {
fireLoaderEvent(loadType, "Processing " + product.getRate(),
isDone);
for (String phuc : hucsToLoad) {
getMonitor().processUri(isProductLoad, rateURI, siteKey,
product.getRate(), timeBack, phuc);
} }
fireLoaderEvent(loadType, product.getRate(), isDone);
}
if (phuc.equals("VIRTUAL")) { // qpes
// we don't do the virtuals like this fireLoaderEvent(loadType, "Processing " + product.getQpe(), isDone);
continue; FFMPAggregateRecord qpeCache = null;
} else {
// rate
if (rateURI != null) {
fireLoaderEvent(loadType,
"Processing " + product.getRate() + "/" + phuc,
isDone);
getMonitor().processUri(isProductLoad, rateURI, if (loadType == LOADER_TYPE.INITIAL) {
siteKey, product.getRate(), timeBack, phuc);
fireLoaderEvent(loadType, product.getRate() + "/" SourceXML source = getMonitor().getSourceConfig().getSource(
+ phuc, isDone); product.getQpe());
qpeCache = readCacheFile(source, dataKey, wfo);
if (qpeCache != null) {
getMonitor().insertFFMPData(qpeCache, siteKey,
product.getQpe());
}
}
// Use this method of QPE data retrieval if you don't have cache files
if (!qpeURIs.isEmpty() && qpeCache == null) {
for (String phuc : hucsToLoad) {
if (phuc.equals(layer)
|| phuc.equals("ALL")) {
getMonitor().processUris(qpeURIs, isProductLoad,
siteKey, product.getQpe(), timeBack, phuc);
} }
}
}
// qpes fireLoaderEvent(loadType, product.getQpe(), isDone);
fireLoaderEvent(loadType, "Processing " + product.getQpe()
+ "/" + phuc, isDone);
FFMPBasinData qpeData = null;
if (loadType == LOADER_TYPE.INITIAL) { int i = 0;
for (NavigableMap<Date, List<String>> qpfURIs : qpfs) {
// qpf
fireLoaderEvent(loadType, "Processing " + product.getQpf(i),
isDone);
FFMPAggregateRecord qpfCache = null;
SourceXML source = getMonitor().getSourceConfig() if (loadType == LOADER_TYPE.INITIAL) {
.getSource(product.getQpe());
qpeData = readLoaderBuddyFile(source, phuc, dataKey, SourceXML source = getMonitor().getSourceConfig()
wfo); .getSource(qpfSources.get(i));
if (qpeData != null) { String pdataKey = findQPFHomeDataKey(source);
qpfCache = readCacheFile(source, pdataKey, wfo);
getMonitor().insertFFMPData(qpeData, siteKey, if (qpfCache != null) {
product.getQpe(), phuc); for (String phuc : hucsToLoad) {
} if ((phuc.equals(layer) || phuc.equals("ALL"))
} && loadType == LOADER_TYPE.INITIAL
&& source.getSourceName().equals(
config.getFFMPConfigData()
.getIncludedQPF())) {
if (!qpfURIs.isEmpty()) {
if (!qpeURIs.isEmpty() && qpeData == null) { getMonitor().processUris(qpfURIs,
if (phuc.equals(config.getFFMPConfigData().getLayer()) isProductLoad, siteKey,
|| phuc.equals("ALL")) { source.getSourceName(), timeBack,
getMonitor().processUris(qpeURIs, isProductLoad, phuc);
siteKey, product.getQpe(), timeBack, phuc);
}
}
fireLoaderEvent(loadType, product.getQpe() + "/" + phuc,
isDone);
int i = 0;
for (NavigableMap<Date, List<String>> qpfURIs : qpfs) {
// qpf
fireLoaderEvent(loadType,
"Processing " + product.getQpf(i) + "/" + phuc,
isDone);
FFMPBasinData qpfData = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig()
.getSource(qpfSources.get(i));
String pdataKey = findQPFHomeDataKey(source);
qpfData = readLoaderBuddyFile(source, phuc,
pdataKey, wfo);
if (qpfData != null) {
if ((phuc.equals(config.getFFMPConfigData()
.getLayer()) || phuc.equals("ALL"))
&& loadType == LOADER_TYPE.INITIAL
&& source.getSourceName().equals(
config.getFFMPConfigData()
.getIncludedQPF())) {
if (!qpfURIs.isEmpty()) {
getMonitor().processUris(qpfURIs,
isProductLoad, siteKey,
source.getSourceName(),
timeBack, phuc);
}
} }
getMonitor().insertFFMPData(qpfData, siteKey,
source.getSourceName(), phuc);
}
}
// if (isUrisProcessNeeded(qpfData,qpfURIs))
// {/*DR13839*/
if ((qpfData == null) && !qpfURIs.isEmpty()) {
if (phuc.equals(config.getFFMPConfigData()
.getLayer()) || phuc.equals("ALL")) { // old
// code:
// keep
// for
// reference*/
// if (isHucProcessNeeded(phuc)) {/*DR13839*/
getMonitor().processUris(qpfURIs,
isProductLoad, siteKey,
product.getQpf(i), timeBack, phuc);
} }
} }
fireLoaderEvent(loadType, product.getQpf(i) + "/" getMonitor().insertFFMPData(qpfCache, siteKey,
+ phuc, isDone); source.getSourceName());
i++;
} }
} }
// virtuals only have data for ALL // if (isUrisProcessNeeded(qpfData,qpfURIs))
if (phuc.equals("ALL")) { // {/*DR13839*/
// Use this method of QPF data retrieval if you don't have cache files
if ((qpfCache == null) && !qpfURIs.isEmpty()) {
for (String phuc : hucsToLoad) {
if (phuc.equals(layer)
|| phuc.equals("ALL")) { // old
// code:
// keep
// for
// reference*/
// if (isHucProcessNeeded(phuc)) {/*DR13839*/
getMonitor().processUris(qpfURIs, isProductLoad,
siteKey, product.getQpf(i), timeBack, phuc);
}
}
}
fireLoaderEvent(loadType, product.getQpf(i), isDone);
i++;
}
fireLoaderEvent(loadType, "Processing " + product.getVirtual(),
isDone);
FFMPAggregateRecord vgbCache = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig().getSource(
product.getVirtual());
vgbCache = readCacheFile(source, dataKey, wfo);
if (vgbCache != null) {
getMonitor().insertFFMPData(vgbCache, siteKey,
product.getVirtual());
}
}
// Use this method of Virtual data retrieval if you don't have cache files
if ((vgbCache == null) && !virtualURIs.isEmpty()) {
getMonitor().processUris(virtualURIs, isProductLoad, siteKey,
product.getVirtual(), timeBack, "ALL");
}
fireLoaderEvent(loadType, product.getVirtual(), isDone);
// process guidance all for all only, never uses cache files
for (String type : productRun.getGuidanceTypes(product)) {
ArrayList<SourceXML> guidSources = productRun
.getGuidanceSources(product, type);
for (SourceXML guidSource : guidSources) {
NavigableMap<Date, List<String>> iguidURIs = guids
.get(guidSource.getSourceName());
fireLoaderEvent(loadType, fireLoaderEvent(loadType,
"Processing " + product.getVirtual() + "/" + phuc, "Processing " + guidSource.getSourceName(), isDone);
getMonitor().processUris(iguidURIs, isProductLoad, siteKey,
guidSource.getSourceName(), timeBack, "ALL");
fireLoaderEvent(loadType, guidSource.getSourceName(),
isDone); isDone);
FFMPBasinData vgbData = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig()
.getSource(product.getVirtual());
vgbData = readLoaderBuddyFile(source, phuc, dataKey,
wfo);
if (vgbData != null) {
getMonitor().insertFFMPData(vgbData, siteKey,
product.getVirtual(), phuc);
}
}
if ((vgbData == null) && !virtualURIs.isEmpty()) {
getMonitor().processUris(virtualURIs, isProductLoad,
siteKey, product.getVirtual(), timeBack, phuc);
}
fireLoaderEvent(loadType,
product.getVirtual() + "/" + phuc, isDone);
} }
// process guidance all at once
for (String type : productRun.getGuidanceTypes(product)) {
ArrayList<SourceXML> guidSources = productRun
.getGuidanceSources(product, type);
for (SourceXML guidSource : guidSources) {
NavigableMap<Date, List<String>> iguidURIs = guids
.get(guidSource.getSourceName());
fireLoaderEvent(loadType,
"Processing " + guidSource.getSourceName()
+ "/" + phuc, isDone);
getMonitor().processUris(iguidURIs, isProductLoad,
siteKey, guidSource.getSourceName(), timeBack,
phuc);
fireLoaderEvent(loadType, guidSource.getSourceName()
+ "/" + phuc, isDone);
}
}
fireLoaderEvent(loadType, phuc + " Load complete", isDone);
} }
} catch (Exception e) { } catch (Exception e) {
System.err.println("FFMP Data Loader terminated...." statusHandler.handle(Priority.PROBLEM,"General Problem in Loading FFMP Data", e);
+ e.getMessage());
} finally { } finally {
isDone = true; isDone = true;
} }
@ -410,7 +395,7 @@ public class FFMPDataLoader extends Thread {
} }
long endTime = (System.currentTimeMillis()) - time; long endTime = (System.currentTimeMillis()) - time;
System.out.println("Loader took: " + endTime / 1000 + " seconds"); System.out.println(loadType.loaderType + " Loader took: " + endTime / 1000 + " seconds");
fireLoaderEvent(loadType, message, isDone); fireLoaderEvent(loadType, message, isDone);
} }
@ -467,19 +452,19 @@ public class FFMPDataLoader extends Thread {
}; };
/** /**
* Loads the loader buddy files * Loads the Cache files
* *
* @param sourceName * @param sourceName
* @param huc * @param huc
* @param wfo * @param wfo
* @return * @return
*/ */
private FFMPBasinData readLoaderBuddyFile(SourceXML source, String huc, private FFMPAggregateRecord readCacheFile(SourceXML source, String pdataKey, String wfo) {
String pdataKey, String wfo) {
long time = System.currentTimeMillis();
String sourceName = source.getSourceName(); String sourceName = source.getSourceName();
File file = new File(sharePath + wfo + File.separator + sourceName File file = new File(sharePath + wfo + File.separator + sourceName
+ "-" + siteKey + "-" + pdataKey + "-" + huc + ".bin"); + "-" + siteKey + "-" + pdataKey + ".bin");
File lockFile = new File(sharePath + wfo + File.separator + sourceName File lockFile = new File(sharePath + wfo + File.separator + sourceName
+ "-" + siteKey + "-" + pdataKey + ".lock"); + "-" + siteKey + "-" + pdataKey + ".lock");
@ -489,22 +474,22 @@ public class FFMPDataLoader extends Thread {
sleep(100); sleep(100);
i++; i++;
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); statusHandler.handle(Priority.ERROR,"Took to long to load Cache Record", e);
} }
} }
break; break;
} }
System.out.println("Buddy File expected path: " System.out.println("Cache File expected path: "
+ file.getAbsolutePath()); + file.getAbsolutePath());
FFMPBasinData basinData = null; FFMPAggregateRecord cacheRecord = null;
if (file.exists()) { if (file.exists()) {
System.out.println("Last mod: " + new Date(file.lastModified())); System.out.println("Last mod: " + new Date(file.lastModified()));
if (file.lastModified() > (System.currentTimeMillis() - (6 * 1000 * 3600))) { if (file.lastModified() > (System.currentTimeMillis() - (6 * TimeUtil.MILLIS_PER_HOUR))) {
while (lockFile.exists()) { while (lockFile.exists()) {
for (int i = 0; i < 4; i++) { for (int i = 0; i < 4; i++) {
@ -514,39 +499,42 @@ public class FFMPDataLoader extends Thread {
sleep(100); sleep(100);
i++; i++;
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); statusHandler.handle(Priority.ERROR,"ERROR in Loading Cache Record", e);
} }
} }
break; break;
} }
BufferedInputStream is = null; GZIPInputStream gis = null;
try { try {
System.out.println("Loading file: " + file.getName()); System.out.println("Loading file: " + file.getName());
is = new BufferedInputStream(new FileInputStream(file)); gis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)));
DynamicSerializationManager dsm = DynamicSerializationManager DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift); .getManager(SerializationType.Thrift);
basinData = (FFMPBasinData) dsm.deserialize(is); cacheRecord = (FFMPAggregateRecord) dsm.deserialize(gis);
} catch (SerializationException e) { } catch (SerializationException e) {
e.printStackTrace(); statusHandler.handle(Priority.ERROR,"Serialization ERROR in Loading Cache Record", e);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); statusHandler.handle(Priority.ERROR,"IO ERROR in Loading Cache Record", e);
} finally { } finally {
if (is != null) { if (gis != null) {
try { try {
is.close(); gis.close();
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); statusHandler.handle(Priority.ERROR,"GENRAL ERROR in Loading Cache Record", e);
} }
} }
} }
} }
} }
long time2 = System.currentTimeMillis();
System.out.println("FFMP Cache file Load took: "+ (time2 - time) + " ms");
return basinData; return cacheRecord;
} }
@ -564,7 +552,7 @@ public class FFMPDataLoader extends Thread {
for (ProductRunXML product : runManager.getProducts()) { for (ProductRunXML product : runManager.getProducts()) {
File file = new File(sharePath + wfo + File.separator File file = new File(sharePath + wfo + File.separator
+ source.getSourceName() + "-" + siteKey + "-" + source.getSourceName() + "-" + siteKey + "-"
+ product.getProductKey() + "-ALL.bin"); + product.getProductKey() + ".bin");
if (file.exists()) { if (file.exists()) {
return product.getProductKey(); return product.getProductKey();
@ -572,7 +560,6 @@ public class FFMPDataLoader extends Thread {
} }
return siteKey; return siteKey;
} }
} }

6
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java
View file

@ -153,8 +153,8 @@ import com.vividsolutions.jts.geom.Point;
* 31 July 2012 14517 mpduff Fix for blanking map on update. * 31 July 2012 14517 mpduff Fix for blanking map on update.
* 14 Sep 2012 1048 njensen Code cleanup * 14 Sep 2012 1048 njensen Code cleanup
* 07 Dec 2012 1353 rferrel Changes for non-blocking FFMPSplash dialog. * 07 Dec 2012 1353 rferrel Changes for non-blocking FFMPSplash dialog.
* 10 Jan 2103 1475 dhladky Some cleanup * 10 Jan 2013 1475 dhladky Some cleanup
* * 27 Jan 2013 1478 dhladky Changed gap collection to a generic list insted of Arraylist
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -3940,7 +3940,7 @@ public class FFMPResource extends
* *
* @return Array of Gap data * @return Array of Gap data
*/ */
public ArrayList<FFMPGap> getGaps() { public List<FFMPGap> getGaps() {
synchronized (timeOrderedKeys) { synchronized (timeOrderedKeys) {
return FFMPGap.getGaps(getTimeOrderedKeys(), getResourceData() return FFMPGap.getGaps(getTimeOrderedKeys(), getResourceData()
.getPrimarySourceXML().getExpirationMinutes(getSiteKey()), .getPrimarySourceXML().getExpirationMinutes(getSiteKey()),

8
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/META-INF/services/com.raytheon.uf.common.serialization.ISerializableObject
View file

@ -1,8 +1,2 @@
com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord
com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPVirtualGageBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData
com.raytheon.uf.common.dataplugin.ffmp.SourceBinList
com.raytheon.uf.common.dataplugin.ffmp.SourceBin
com.raytheon.uf.common.dataplugin.ffmp.SourceBinEntry

104
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java Normal file
View file

@ -0,0 +1,104 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Cache Record implementation for FFMP plugin
* Eventually this will become a full record implementation
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Created to reduce memory and disk read/writes for FFMP
*
* </pre>
*
* @author dhladky
* @version 1
*/
@DynamicSerialize
public class FFMPAggregateRecord implements ISerializableObject {
private static final long serialVersionUID = 76774564363471L;
/**
*
*/
public FFMPAggregateRecord() {
}
@DynamicSerializeElement
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
@DynamicSerializeElement
private List<Long> times = new ArrayList<Long>();
public void setTimes(List<Long> times) {
this.times = times;
}
public List<Long> getTimes() {
return times;
}
public void setBasinsMap(HashMap<String, FFMPBasinData> basinsMap) {
this.basinsMap = basinsMap;
}
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Add a basin Data Cache object
* @param cacheData
*/
public void setBasinData(FFMPBasinData cacheData) {
basinsMap.put(cacheData.getHucLevel(), cacheData);
}
/**
* Gets the BasinData object
* @param huc
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinsMap.containsKey(huc)) {
return basinsMap.get(huc);
}
return null;
}
}

87
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java
View file

@ -22,14 +22,11 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.TreeMap; import java.util.TreeMap;
import javax.xml.bind.annotation.XmlAccessType; import javax.persistence.Transient;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
@ -45,30 +42,49 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 06/22/09 2152 D. Hladky Initial release * 06/22/09 2152 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added support for writing aggregate record cache
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1 * @version 1
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class FFMPBasin implements ISerializableObject, Cloneable { public class FFMPBasin implements ISerializableObject, Cloneable {
/** pfafstetter id(key) in GIS **/ /** pfafstetter id(key) in GIS **/
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
protected Long pfaf; protected Long pfaf;
/** boolean aggregator **/ /** boolean aggregator **/
@DynamicSerializeElement @DynamicSerializeElement
@XmlAttribute
protected boolean aggregated = false; protected boolean aggregated = false;
@DynamicSerializeElement /** object used in calculations
@XmlElement * not serialized
**/
@Transient
protected TreeMap<Date, Float> values; protected TreeMap<Date, Float> values;
/** object used for serialization **/
@DynamicSerializeElement
public float[] cacheValues;
/**
* Get the float array of serialized values
* @return
*/
public float[] getCacheValues() {
return cacheValues;
}
/**
* Set the serialized array of cache values
* @param cacheValues
*/
public void setCacheValues(float[] cacheValues) {
this.cacheValues = cacheValues;
}
/** /**
* @return the pfaf_id * @return the pfaf_id
@ -326,14 +342,23 @@ public class FFMPBasin implements ISerializableObject, Cloneable {
} }
/** /**
* No arg hibernate constructor * No arg serial constructor
*/ */
public FFMPBasin() { public FFMPBasin() {
values = new TreeMap<Date, Float>(new Comparator<Date>() {
@Override
public int compare(Date o1, Date o2) {
// Null checks?
return (int)(o2.getTime() - o1.getTime()) ;
}
});
} }
/** /**
* useful constructor * Useful constructor
* @param pfaf
* @param aggregated
*/ */
public FFMPBasin(Long pfaf, boolean aggregated) { public FFMPBasin(Long pfaf, boolean aggregated) {
setPfaf(pfaf); setPfaf(pfaf);
@ -342,11 +367,45 @@ public class FFMPBasin implements ISerializableObject, Cloneable {
@Override @Override
public int compare(Date o1, Date o2) { public int compare(Date o1, Date o2) {
// Null checks? // Null checks?
return (int)Math.signum(o2.getTime() - o1.getTime()) ; return (int)(o2.getTime() - o1.getTime()) ;
} }
}); });
} }
/**
* Populates the values from the cache
*
* @param times
*/
public void populate(List<Long> times) {
// safe to avoid Array Index Exceptions / shouldn't happen but.....
if (cacheValues != null && (times.size() == cacheValues.length)) {
int i = 0;
for (Long time : times) {
values.put(new Date(time), cacheValues[i]);
i++;
}
//System.out.println("populated :"+i+" pfaf : "+pfaf);
}
}
/**
* populates the serialized array
*/
public void setCache() {
cacheValues = new float[values.size()];
int i = 0;
for (Date date: values.descendingKeySet()) {
cacheValues[i] = values.get(date);
i++;
}
//System.out.println("wrote :"+i+" pfaf : "+pfaf);
}
/** /**
* purge out old entries * purge out old entries

35
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java
View file

@ -22,11 +22,7 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
@ -42,14 +38,13 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 06/22/09 2152 D. Hladky Initial release * 06/22/09 2152 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added support for write of aggregate record cache
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1 * @version 1
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class FFMPBasinData implements ISerializableObject { public class FFMPBasinData implements ISerializableObject {
@ -62,11 +57,9 @@ public class FFMPBasinData implements ISerializableObject {
// defaults // defaults
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
private String hucLevel; private String hucLevel;
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
private HashMap<Long, FFMPBasin> basins = new HashMap<Long, FFMPBasin>(); private HashMap<Long, FFMPBasin> basins = new HashMap<Long, FFMPBasin>();
/** /**
@ -86,7 +79,7 @@ public class FFMPBasinData implements ISerializableObject {
} }
/** /**
* Used only internally * get the basin map
* *
* @return * @return
*/ */
@ -95,7 +88,7 @@ public class FFMPBasinData implements ISerializableObject {
} }
/** /**
* DONT USE THIS EVER!!!!! * Sets the basin map
* *
* @param basins * @param basins
*/ */
@ -545,5 +538,25 @@ public class FFMPBasinData implements ISerializableObject {
basin.purgeData(date); basin.purgeData(date);
} }
} }
/**
* populates data from the cache
*
* @param times
*/
public void populate(List<Long> times) {
for (FFMPBasin basin : basins.values()) {
basin.populate(times);
}
}
/**
* populates the serialized array/objects
*/
public void setCache() {
for (FFMPBasin basin : basins.values()) {
basin.setCache();
}
}
} }

28
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPCacheRecord.java
View file

@ -1,4 +1,23 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.awt.Point; import java.awt.Point;
import java.io.IOException; import java.io.IOException;
@ -210,16 +229,16 @@ public class FFMPCacheRecord extends FFMPRecord {
} }
/** /**
* Buddy File reader * Cache File reader
* *
* @param basins * @param basins
* @param hucName * @param hucName
*/ */
public void setBasinBuddyData(FFMPBasinData basins, String hucName) { public void setCacheData(FFMPBasinData basins, String hucName) {
if (getBasinData(hucName) != null) { if (getBasinData(hucName) != null) {
basins = getBasinData(hucName, true); basins = getBasinData(hucName, true);
//System.out.println("Adding pieces Buddy Data: "+hucName+" "+getSourceName()); //System.out.println("Adding Cache Data: "+hucName+" "+getSourceName());
synchronized (basins) { synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.getBasins() for (Entry<Long, FFMPBasin> entry : basins.getBasins()
@ -442,8 +461,7 @@ public class FFMPCacheRecord extends FFMPRecord {
} catch (Throwable e) { } catch (Throwable e) {
statusHandler.handle(Priority.PROBLEM, statusHandler.handle(Priority.PROBLEM,
"ERROR Retrieving Map for URI: " + uri "ERROR Retrieving Map for URI: " + uri
+ "..." + huc); + "..." + huc, e);
e.printStackTrace();
} }
} }
} }

119
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java
View file

@ -20,9 +20,9 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@ -33,7 +33,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
/** /**
* FFTI Data Container * FFMP Data Container
* *
* <pre> * <pre>
* *
@ -43,7 +43,8 @@ import com.raytheon.uf.common.status.UFStatus;
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 03/31/11 5489 D. Hladky Initial release * 03/31/11 5489 D. Hladky Initial release
* 07/31/12 578 D.Hladky finished it * 07/31/12 578 D.Hladky finished it
* 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException * 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException
* 01/27/13 1478 D. Hladky Re-worked to help with memory size and NAS read write stress
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -62,15 +63,24 @@ public class FFMPDataContainer {
private String filePath = null; private String filePath = null;
public FFMPDataContainer() { public FFMPDataContainer() {
// public unused constructor
} }
/**
* Usual constructor
* @param sourceName
*/
public FFMPDataContainer(String sourceName) { public FFMPDataContainer(String sourceName) {
this.sourceName = sourceName; this.sourceName = sourceName;
basinDataMap.put("ALL", new FFMPBasinData("ALL")); basinDataMap.put("ALL", new FFMPBasinData("ALL"));
// System.out.println("Creating source: " + sourceName); // System.out.println("Creating source: " + sourceName);
} }
/**
* special constuctor
* @param sourceName
* @param hucs
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs) { public FFMPDataContainer(String sourceName, ArrayList<String> hucs) {
// System.out.println("Creating source with hucs: " + sourceName); // System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName; this.sourceName = sourceName;
@ -78,6 +88,21 @@ public class FFMPDataContainer {
basinDataMap.put(huc, new FFMPBasinData(huc)); basinDataMap.put(huc, new FFMPBasinData(huc));
} }
} }
/**
* new container first time read in from cache
* @param sourceName
* @param hucs
* @param record
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs, FFMPAggregateRecord record) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
FFMPBasinData basinData = record.getBasinData(huc);
basinData.populate(record.getTimes());
basinDataMap.put(huc, basinData);
}
}
/** /**
* Adds to the cache * Adds to the cache
@ -400,7 +425,7 @@ public class FFMPDataContainer {
* @param barrierTime * @param barrierTime
* @return * @return
*/ */
public ArrayList<Date> getOrderedTimes(Date barrierTime) { public List<Date> getOrderedTimes(Date barrierTime) {
ArrayList<Date> orderedTimes = new ArrayList<Date>(); ArrayList<Date> orderedTimes = new ArrayList<Date>();
try { try {
HashMap<Long, FFMPBasin> basins = getBasinData("ALL").getBasins(); HashMap<Long, FFMPBasin> basins = getBasinData("ALL").getBasins();
@ -414,7 +439,32 @@ public class FFMPDataContainer {
} }
} }
Collections.reverse(orderedTimes); return orderedTimes;
}
}
} catch (Exception e) {
statusHandler.debug("No ordered times available..."
+ getSourceName());
}
return null;
}
/**
* Gets the list of times for serialization
* @return
*/
public List<Long> getOrderedTimes() {
ArrayList<Long> orderedTimes = new ArrayList<Long>();
try {
HashMap<Long, FFMPBasin> basins = getBasinData("ALL").getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
FFMPBasin basin = entry.getValue();
for (Date time : basin.getValues().descendingKeySet()) {
orderedTimes.add(time.getTime());
}
return orderedTimes; return orderedTimes;
} }
@ -422,18 +472,22 @@ public class FFMPDataContainer {
} catch (Exception e) { } catch (Exception e) {
statusHandler.debug("No ordered times available..." statusHandler.debug("No ordered times available..."
+ getSourceName()); + getSourceName());
return null;
} }
return null; return null;
} }
/**
* Gets the source name for this Data Container
* @return
*/
public String getSourceName() { public String getSourceName() {
return sourceName; return sourceName;
} }
/* /**
* clean up old junk * Clean up old junk
* @param backDate
*/ */
public void purge(Date backDate) { public void purge(Date backDate) {
for (String huc : basinDataMap.keySet()) { for (String huc : basinDataMap.keySet()) {
@ -442,27 +496,33 @@ public class FFMPDataContainer {
} }
/** /**
* maybe this will work * Sets the Cache data for this container
* *
* @param basins * @param cacheRecord
* @param hucName
*/ */
public void setBasinBuddyData(FFMPBasinData basins, String hucName) { public void setCacheData(FFMPAggregateRecord cacheRecord) {
for (Entry<Long, FFMPBasin> entry : basins.getBasins().entrySet()) { // create a record from the cache record
FFMPBasin basin = getBasinData(hucName).get(entry.getKey()); FFMPRecord record = new FFMPRecord(cacheRecord);
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) { for (Entry<String, FFMPBasinData> dentry : record.getBasinsMap()
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin; .entrySet()) {
gbasin.getGuidValues().putAll( for (Entry<Long, FFMPBasin> entry : dentry.getValue().getBasins()
((FFMPGuidanceBasin) entry.getValue()) .entrySet()) {
.getGuidValues()); FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
basin.getValues().putAll(entry.getValue().getValues());
}
} else { } else {
basin.getValues().putAll(entry.getValue().getValues()); syncPut(getBasinData(dentry.getKey()), entry.getKey(),
entry.getValue());
} }
} else {
syncPut(getBasinData(hucName), entry.getKey(), entry.getValue());
//getBasinData(hucName).put(entry.getKey(), entry.getValue());
} }
} }
} }
@ -523,4 +583,13 @@ public class FFMPDataContainer {
basins.put(key, value); basins.put(key, value);
} }
} }
/**
* Gets the basin data map
* @return
*/
public ConcurrentHashMap<String, FFMPBasinData> getBasinMap() {
return basinDataMap;
}
} }

39
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGap.java
View file

@ -1,4 +1,23 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
/** /**
* gap for FFMP * gap for FFMP
@ -10,6 +29,7 @@ package com.raytheon.uf.common.dataplugin.ffmp;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 03/03/11 7334 D. Hladky Initial release * 03/03/11 7334 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added use of constants for calculations
* *
* </pre> * </pre>
* *
@ -19,6 +39,9 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List;
import com.raytheon.uf.common.time.util.TimeUtil;
public class FFMPGap { public class FFMPGap {
@ -62,21 +85,25 @@ public class FFMPGap {
} }
/** /**
* Gets the GAP calculation for an FFMP source * Get the gaps in the FFMP data
* *
* @return Array of Gap data * @param times
* @param expirationTime
* @param barrierTime
* @param mostRecentTime
* @return
*/ */
public static ArrayList<FFMPGap> getGaps(ArrayList<Date> times, public static List<FFMPGap> getGaps(List<Date> times,
long expirationTime, Date barrierTime, Date mostRecentTime) { long expirationTime, Date barrierTime, Date mostRecentTime) {
ArrayList<FFMPGap> gaps = new ArrayList<FFMPGap>(); ArrayList<FFMPGap> gaps = new ArrayList<FFMPGap>();
long gapStep = expirationTime * 60 * 1000; long gapStep = expirationTime * TimeUtil.MILLIS_PER_MINUTE;
Date prevTime = null; Date prevTime = null;
// System.out.println("Calling getGaps()...Recent Time: " + mostRecentTime // System.out.println("Calling getGaps()...Recent Time: " + mostRecentTime
// + " BarrierTime: " + barrierTime); // + " BarrierTime: " + barrierTime);
if (times.size() == 1) { if (times.size() == 1) {
FFMPGap gap = new FFMPGap(); FFMPGap gap = new FFMPGap();
long totalMillis = mostRecentTime.getTime() - barrierTime.getTime() - gapStep; long totalMillis = mostRecentTime.getTime() - barrierTime.getTime() - gapStep;
float gapMinutes = (totalMillis)/(60 * 1000); float gapMinutes = totalMillis/TimeUtil.MILLIS_PER_MINUTE;
if (gapMinutes < 0.0) { if (gapMinutes < 0.0) {
gapMinutes = 0.0f; gapMinutes = 0.0f;
} }
@ -100,7 +127,7 @@ public class FFMPGap {
FFMPGap gap = new FFMPGap(prevTime, time); FFMPGap gap = new FFMPGap(prevTime, time);
// convert to minutes and set gap // convert to minutes and set gap
// Need to subtract the expirationTime from the gapTime as well // Need to subtract the expirationTime from the gapTime as well
gap.setGap((gapTime - gapStep) / (60 * 1000)); gap.setGap((gapTime - gapStep) / TimeUtil.MILLIS_PER_MINUTE);
gaps.add(gap); gaps.add(gap);
} }
prevTime = time; prevTime = time;

27
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGuidanceBasin.java
View file

@ -4,13 +4,10 @@ import java.util.ArrayList;
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List;
import java.util.TreeMap; import java.util.TreeMap;
import javax.persistence.Transient; import javax.persistence.Transient;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.monitor.config.FFFGDataMgr; import com.raytheon.uf.common.monitor.config.FFFGDataMgr;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
@ -27,22 +24,21 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 08/22/10 3437 D. Hladky Initial release * 08/22/10 3437 D. Hladky Initial release
* 01/17/13 1478 D. Hladky Removed un-needed XML attributes
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1 * @version 1
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject { public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject {
public FFMPGuidanceBasin() { public FFMPGuidanceBasin() {
} }
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
protected TreeMap<Date, HashMap<String, Float>> guidValues; protected TreeMap<Date, HashMap<String, Float>> guidValues;
@Transient @Transient
@ -411,7 +407,10 @@ public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject
} }
/** /**
* useful constructor * Constructor used in producing a new GuidanceBasin
*
* @param pfaf
* @param aggregated
*/ */
public FFMPGuidanceBasin(Long pfaf, boolean aggregated) { public FFMPGuidanceBasin(Long pfaf, boolean aggregated) {
setPfaf(pfaf); setPfaf(pfaf);
@ -457,5 +456,15 @@ public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject
} }
return buff.toString(); return buff.toString();
} }
public void populate(List<Long> times) {
// does nothing here, don't need to populate anything.
}
/**
* populates the serialized array
*/
public void setCache() {
// does nothing here, this class is serialized as is.
}
} }

4
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGuidanceInterpolation.java
View file

@ -30,6 +30,7 @@ import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager;
import com.raytheon.uf.common.monitor.xml.ProductRunXML; import com.raytheon.uf.common.monitor.xml.ProductRunXML;
import com.raytheon.uf.common.monitor.xml.ProductXML; import com.raytheon.uf.common.monitor.xml.ProductXML;
import com.raytheon.uf.common.monitor.xml.SourceXML; import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.time.util.TimeUtil;
/** /**
* Guidance Interpolation * Guidance Interpolation
@ -39,6 +40,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 29 Jan, 2010 3915 dhladky Initial creation * 29 Jan, 2010 3915 dhladky Initial creation
* 01/27/13 1478 dhladky Added use of constants
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -258,7 +260,7 @@ public class FFMPGuidanceInterpolation {
getSource(orderedHours.get(index)), getSource(orderedHours.get(index)),
null, null,
manager.getSource(source).getExpirationMinutes( manager.getSource(source).getExpirationMinutes(
siteKey) * 60 * 1000); siteKey) * TimeUtil.MILLIS_PER_MINUTE);
if (dman.isExpired() == false) { if (dman.isExpired() == false) {
thisVal = dman.adjustValue(thisVal, thisVal = dman.adjustValue(thisVal,

36
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java
View file

@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
import javax.persistence.Column; import javax.persistence.Column;
@ -66,6 +67,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 06/03/09 2521 D. Hladky Initial release * 06/03/09 2521 D. Hladky Initial release
* 01/27/13 1478 D. Hladky OUN memory help
* *
* </pre> * </pre>
* *
@ -758,5 +760,39 @@ public class FFMPRecord extends ServerSpecificPersistablePluginDataObject
public String getSiteKey() { public String getSiteKey() {
return siteKey; return siteKey;
} }
/**
* Get the fully cache ready object
* @param fileName
* @return
*/
public FFMPAggregateRecord getCacheRecord() {
FFMPAggregateRecord fdcr = new FFMPAggregateRecord();
for (Entry<String,FFMPBasinData> entry: basinsMap.entrySet()) {
fdcr.setBasinData(entry.getValue());
}
return fdcr;
}
/**
* Creates and populates a version of this record from a cache record
*
* @param fdcr
*/
public FFMPRecord(FFMPAggregateRecord fdcr) {
List<Long> times = fdcr.getTimes();
for (Entry<String, FFMPBasinData> entry : fdcr.getBasinsMap()
.entrySet()) {
FFMPBasinData fbd = entry.getValue();
// Keep in mind times can be null, Guidance basins are like that
fbd.populate(times);
setBasinData(fbd, fbd.getHucLevel());
}
}
} }

26
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPVirtualGageBasin.java
View file

@ -1,13 +1,28 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.Comparator; import java.util.Comparator;
import java.util.Date; import java.util.Date;
import java.util.TreeMap; import java.util.TreeMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -21,14 +36,13 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 02may10 3937 dhladky Setup * 02may10 3937 dhladky Setup
* 01/27/13 1478 dhladky Removed un-needed XML annotations
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1.0 * @version 1.0
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class FFMPVirtualGageBasin extends FFMPBasin implements public class FFMPVirtualGageBasin extends FFMPBasin implements
ISerializableObject { ISerializableObject {

31
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBin.java
View file

@ -1,12 +1,26 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList; import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -22,28 +36,25 @@ import com.vividsolutions.jts.geom.Coordinate;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 10/22/10 6581 D. Hladky Initial release * 10/22/10 6581 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1 * @version 1
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class SourceBin implements ISerializableObject { public class SourceBin implements ISerializableObject {
/** sourceName and dataKey **/ /** sourceName and dataKey **/
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public double[] lats; public double[] lats;
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public double[] lons; public double[] lons;
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public double[] areaPercent; public double[] areaPercent;
public SourceBin() { public SourceBin() {

43
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBinEntry.java
View file

@ -1,24 +1,53 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
import javax.xml.bind.annotation.XmlAccessType; * This software was developed and / or modified by Raytheon Company,
import javax.xml.bind.annotation.XmlAccessorType; * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
import javax.xml.bind.annotation.XmlElement; *
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.vividsolutions.jts.geom.Coordinate; import com.vividsolutions.jts.geom.Coordinate;
/**
* FFMP source binning entry object
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
*
* </pre>
*
* @author dhladky
* @version 1
*/
@DynamicSerialize @DynamicSerialize
@XmlAccessorType(XmlAccessType.NONE)
public class SourceBinEntry { public class SourceBinEntry {
/** sourceName and dataKey **/ /** sourceName and dataKey **/
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public Coordinate coor; public Coordinate coor;
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public double area; public double area;
public Coordinate getCoor() { public Coordinate getCoor() {

29
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBinList.java
View file

@ -1,12 +1,26 @@
package com.raytheon.uf.common.dataplugin.ffmp; package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.HashMap; import java.util.HashMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject; import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -22,24 +36,21 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 10/22/10 6581 D. Hladky Initial release * 10/22/10 6581 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
* *
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
* @version 1 * @version 1
*/ */
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize @DynamicSerialize
public class SourceBinList implements ISerializableObject { public class SourceBinList implements ISerializableObject {
/** sourceName and dataKey **/ /** sourceName and dataKey **/
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public String sourceId; public String sourceId;
@DynamicSerializeElement @DynamicSerializeElement
@XmlElement
public HashMap<Long, SourceBin> sourceMap; public HashMap<Long, SourceBin> sourceMap;
public SourceBinList() { public SourceBinList() {

4
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml
View file

@ -14,8 +14,8 @@
</bean> </bean>
<bean id="ffmpProcessThreadPool" <bean id="ffmpProcessThreadPool"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor"> class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="3" /> <property name="corePoolSize" value="2" />
<property name="maxPoolSize" value="5" /> <property name="maxPoolSize" value="4" />
<property name="keepAliveSeconds" value="60000" /> <property name="keepAliveSeconds" value="60000" />
</bean> </bean>

294
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -31,15 +31,19 @@ import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map.Entry;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor; import java.util.concurrent.Executor;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import com.raytheon.edex.msg.DataURINotificationMessage; import com.raytheon.edex.msg.DataURINotificationMessage;
import com.raytheon.edex.plugin.radar.dao.RadarStationDao; import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
import com.raytheon.edex.urifilter.URIFilter; import com.raytheon.edex.urifilter.URIFilter;
import com.raytheon.edex.urifilter.URIGenerateMessage; import com.raytheon.edex.urifilter.URIGenerateMessage;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData; import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPDataContainer; import com.raytheon.uf.common.dataplugin.ffmp.FFMPDataContainer;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord; import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
@ -79,6 +83,7 @@ import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.FileUtil; import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.core.EDEXUtil; import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.props.PropertiesFactory; import com.raytheon.uf.edex.core.props.PropertiesFactory;
@ -105,6 +110,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIProcessor;
* 06/21/2009 2521 dhladky Initial Creation. * 06/21/2009 2521 dhladky Initial Creation.
* 02/03/2011 6500 cjeanbap Fixed NullPointerException. * 02/03/2011 6500 cjeanbap Fixed NullPointerException.
* 07/31/2011 578 dhladky FFTI modifications * 07/31/2011 578 dhladky FFTI modifications
* 01/27/13 1478 D. Hladky Added creation of full cache records to help read write stress on NAS
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -134,6 +140,18 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private static final String productType = "ffmp"; private static final String productType = "ffmp";
/**
* The thought was this will eventually be dynamic when We start writing
* long time source records to a DAO. This is the time backward limit for FFTI and cache load data.
*/
public static final int SOURCE_CACHE_TIME = 6;
/**
* The thought was this will eventually be dynamic, static in AWIPS I.
* This is the time back limit for Flash Flood Guidance sources
*/
public static final int FFG_SOURCE_CACHE_TIME = 24;
/** ArrayList of domains to filter for */ /** ArrayList of domains to filter for */
private ArrayList<DomainXML> domains = null; private ArrayList<DomainXML> domains = null;
@ -199,7 +217,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
this.pathManager = PathManagerFactory.getPathManager(); this.pathManager = PathManagerFactory.getPathManager();
statusHandler.handle(Priority.DEBUG, getGeneratorName() statusHandler.handle(Priority.INFO, getGeneratorName()
+ " process Filter Config..."); + " process Filter Config...");
domains = new ArrayList<DomainXML>(); domains = new ArrayList<DomainXML>();
boolean configValid = getRunConfig().isPopulated(); boolean configValid = getRunConfig().isPopulated();
@ -418,7 +436,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} }
} else { } else {
statusHandler statusHandler
.debug(getGeneratorName() .info(getGeneratorName()
+ ": templates not loaded yet. Skipping product"); + ": templates not loaded yet. Skipping product");
} }
} }
@ -500,7 +518,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.DEBUG, statusHandler.handle(Priority.DEBUG,
"Checking status ..." + fftiDone); "Checking status ..." + fftiDone);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); statusHandler.handle(Priority.DEBUG,
"Checking status failed!" + e);
} }
} }
@ -574,16 +593,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements
@Override @Override
public void run() { public void run() {
try { try {
logger.debug("ProcessProduct: Starting thread " statusHandler.handle(
+ ffmpProduct.getSourceName()); Priority.DEBUG,
"ProcessProduct: Starting thread "
+ ffmpProduct.getSourceName());
process(); process();
logger.debug("ProcessProduct: Finishing thread " statusHandler.handle(
+ ffmpProduct.getSourceName()); Priority.DEBUG,
"ProcessProduct: Finishing thread "
+ ffmpProduct.getSourceName());
} catch (Exception e) { } catch (Exception e) {
processes.remove(ffmpProduct.getSourceName()); processes.remove(ffmpProduct.getSourceName());
logger.error("ProcessProduct: removed " statusHandler.handle(Priority.ERROR, "ProcessProduct: removed "
+ ffmpProduct.getSourceName()); + ffmpProduct.getSourceName(), e);
e.printStackTrace();
} }
} }
@ -713,7 +735,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
generator, ffmpRec, generator, ffmpRec,
ffmp.getFFTISource()); ffmp.getFFTISource());
fftiSources.add(ffmp.getFFTISource()); fftiSources.add(ffmp.getFFTISource());
// System.out.println("Adding source to FFTISources!!!!!!!!!!!!"+ffmpRec.getSourceName());
ffti.processFFTI(); ffti.processFFTI();
} }
} }
@ -735,7 +756,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
while (productKeys.size() > 0) { while (productKeys.size() > 0) {
// wait for all threads to finish before returning // wait for all threads to finish before returning
try { try {
Thread.sleep(50); Thread.sleep(100);
statusHandler.handle(Priority.DEBUG, statusHandler.handle(Priority.DEBUG,
"Checking status ..." + productKeys.size()); "Checking status ..." + productKeys.size());
for (String source : productKeys.keySet()) { for (String source : productKeys.keySet()) {
@ -743,7 +764,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
"Still processing ..." + source); "Still processing ..." + source);
} }
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); statusHandler.handle(Priority.WARN,
"Product Procesing Interrupted! " + e);
} }
} }
} }
@ -865,7 +887,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
try { try {
Thread.sleep(100); Thread.sleep(100);
} catch (InterruptedException e) { } catch (InterruptedException e) {
statusHandler.handle(Priority.WARN,
"Domain processing Interrupted!", e);
} }
} }
} }
@ -974,8 +997,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteSourceFileName(sourceId)); getAbsoluteSourceFileName(sourceId));
try { try {
sbl = (SourceBinList) SerializationUtil sbl = SerializationUtil
.transformFromThrift(FileUtil.file2bytes(f.getFile(), true)); .transformFromThrift(SourceBinList.class, FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) { } catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName()); "Unable to locate file " + f.getName());
@ -983,7 +1006,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"Unable to read file " + f.getName()); "Unable to read file " + f.getName());
} catch (IOException ioe) { } catch (IOException ioe) {
ioe.printStackTrace(); statusHandler.handle(Priority.ERROR, "General IO problem with file "
+ f.getName(), ioe);
} }
return sbl; return sbl;
@ -1092,7 +1116,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle( statusHandler.handle(
Priority.ERROR, Priority.ERROR,
"Unable to locate new FFG file. " "Unable to locate new FFG file. "
+ pattern.toString()); + pattern.toString(), e);
} }
} }
} }
@ -1124,11 +1148,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
siteKey = parts[0]; siteKey = parts[0];
} }
container = loadFFMPDataContainer(siteSourceKey, container = loadFFMPDataContainer(siteSourceKey, hucs, siteKey,
config.getCWA(), backDate);
hucs, siteKey,
config.getCWA(), backDate);
if (container != null) { if (container != null) {
ffmpData.put(siteSourceKey, container); ffmpData.put(siteSourceKey, container);
@ -1200,7 +1221,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR, getGeneratorName() statusHandler.handle(Priority.ERROR, getGeneratorName()
+ ": filter: " + filter.getName() + ": filter: " + filter.getName()
+ ": failed to route filter to generator", e); + ": failed to route filter to generator", e);
e.printStackTrace();
} }
filter.setValidTime(new Date(System.currentTimeMillis())); filter.setValidTime(new Date(System.currentTimeMillis()));
@ -1262,7 +1282,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
boolean write = true; boolean write = true;
try { try {
// write out the fast loader buddy file // write out the fast loader cache file
long ptime = System.currentTimeMillis(); long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource( SourceXML source = getSourceConfig().getSource(
@ -1274,15 +1294,16 @@ public class FFMPGenerator extends CompositeProductGenerator implements
sourceName = source.getDisplayName(); sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName; sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day // FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime() backDate = new Date(
- (3600 * 1000 * 24)); config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else { } else {
sourceName = ffmpRec.getSourceName(); sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey() sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey; + "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime() backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime() .getTime()
- (3600 * 1000 * 6)); - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
} }
// deal with setting of needed HUCS // deal with setting of needed HUCS
@ -1336,7 +1357,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (newDate != null && oldDate != null) { if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source .getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * 60 * 1000)) { .getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in // force a re-query back to the newest time in
// existing source container, this will fill in // existing source container, this will fill in
// gaps // gaps
@ -1348,10 +1369,12 @@ public class FFMPGenerator extends CompositeProductGenerator implements
ffmpRec.getWfo(), source, ffmpRec ffmpRec.getWfo(), source, ffmpRec
.getSiteKey()); .getSiteKey());
} else if (oldDate.after(new Date(backDate } else if (oldDate
.getTime() .after(new Date(
- (source.getExpirationMinutes(ffmpRec backDate.getTime()
.getSiteKey()) * 60 * 1000)))) { - (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for // force a re-query back to barrierTime for
// existing source container, this happens if // existing source container, this happens if
// the // the
@ -1423,14 +1446,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// this is defensive for if errors get thrown // this is defensive for if errors get thrown
if (backDate == null) { if (backDate == null) {
backDate = new Date((System.currentTimeMillis()) backDate = new Date((System.currentTimeMillis())
- (3600 * 1000 * 6)); - (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
} }
fdc.purge(backDate); fdc.purge(backDate);
if (write) { if (write) {
// write it out // write it out
writeLoaderBuddyFiles(fdc); writeCacheFiles(fdc);
} }
} }
} }
@ -1451,37 +1474,31 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
FFMPDataContainer fdc = null; FFMPDataContainer fdc = null;
FFMPAggregateRecord record = null;
boolean populated = false;
synchronized (hucs) { if (checkCacheFile(sourceSiteDataKey, wfo, backDate)) {
for (String huc : hucs) { try {
record = readCacheFile(sourceSiteDataKey, wfo, backDate);
FFMPBasinData basinData = null; } catch (Exception e) {
statusHandler.handle(Priority.ERROR,
if (checkBuddyFile(sourceSiteDataKey, huc, wfo, backDate)) { "General Error Reading cache file: " + e.getMessage());
try {
basinData = readLoaderBuddyFile(sourceSiteDataKey, huc,
wfo, backDate);
} catch (Exception e) {
statusHandler.handle(
Priority.ERROR,
"General Error Reading buddy file: "
+ e.getMessage());
}
if (fdc == null) {
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs);
}
}
if (basinData != null) {
fdc.setBasinBuddyData(basinData, huc);
}
} }
if (fdc == null && record != null) {
// creates a place holder for this source
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs, record);
populated = true;
}
}
if (record != null && !populated) {
fdc.setCacheData(record);
} }
if (fdc != null) { if (fdc != null) {
long time2 = System.currentTimeMillis(); long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG, "Loaded Source files: in " statusHandler.handle(Priority.INFO, "Loaded Source files: in "
+ (time2 - time) + " ms: source: " + sourceSiteDataKey); + (time2 - time) + " ms: source: " + sourceSiteDataKey);
} }
@ -1489,60 +1506,59 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} }
/** /**
* Load existing buddy file * Load existing cache file
* *
* @param sourceSiteDataKey * @param sourceSiteDataKey
* @param huc
* @param wfo * @param wfo
* @return * @return
* @throws IOException * @throws IOException
*/ */
private FFMPBasinData readLoaderBuddyFile(String sourceSiteDataKey, private FFMPAggregateRecord readCacheFile(String sourceSiteDataKey, String wfo,
String huc, String wfo, Date backDate) throws IOException { Date backDate) throws IOException {
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + "-" File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + ".bin");
+ huc + ".bin"); FFMPAggregateRecord record = null;
FFMPBasinData basinData = null; GZIPInputStream gis = null;
BufferedInputStream is = null;
try { try {
is = new BufferedInputStream(new FileInputStream(file)); gis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)));
DynamicSerializationManager dsm = DynamicSerializationManager DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift); .getManager(SerializationType.Thrift);
basinData = (FFMPBasinData) dsm.deserialize(is); record = (FFMPAggregateRecord) dsm.deserialize(gis);
} catch (SerializationException e) { } catch (SerializationException e) {
statusHandler statusHandler
.handle(Priority.ERROR, .handle(Priority.ERROR,
"Serialization Error Reading buddy file: " "Serialization Error Reading cache file: "
+ e.getMessage()); + e.getMessage());
} catch (IOException e) { } catch (IOException e) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"IO Error Reading buddy file: " + e.getMessage()); "IO Error Reading cache file: " + e.getMessage());
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"General Error Reading buddy file: " + e.getMessage()); "General Error Reading cache file: " + e.getMessage());
} catch (Throwable t) { } catch (Throwable t) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"Bogus Thrift Error Reading buddy file: " + t.getMessage()); "Bogus Thrift Error Reading cache file: " + t.getMessage());
} finally { } finally {
if (is != null) { if (gis != null) {
is.close(); gis.close();
} }
} }
return basinData; return record;
} }
/** /**
* Write buddy file * Write cache file
* *
* @param sourceSiteDataKey * @param sourceSiteDataKey
* @param huc * @param huc
* @param wfo * @param wfo
* @return * @return
*/ */
public void writeLoaderBuddyFiles(FFMPDataContainer fdc) { public void writeCacheFiles(FFMPDataContainer fdc) {
// Write all huc levels in separate files // Write all huc levels in separate files
File fileDir = new File("" + sharePath + config.getCWA()); File fileDir = new File("" + sharePath + config.getCWA());
@ -1555,7 +1571,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} }
/** /**
* Inner class to thread writing of BuddyFiles * Inner class to thread writing of cache files
* *
* @author dhladky * @author dhladky
* *
@ -1569,7 +1585,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long time = System.currentTimeMillis(); long time = System.currentTimeMillis();
write(); write();
long time2 = System.currentTimeMillis(); long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG, "Wrote loader files: in " statusHandler.handle(Priority.INFO, "Wrote cache file: in "
+ (time2 - time) + " ms :" + fdc.getFilePath()); + (time2 - time) + " ms :" + fdc.getFilePath());
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
@ -1604,61 +1620,69 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (lockfile.canWrite()) { if (lockfile.canWrite()) {
// write the lock if we can even write to anything // write the lock if we can even write to anything
synchronized (fdc.getKeys()) { FFMPAggregateRecord cacheRecord = null;
for (String huc : fdc.getKeys()) {
FFMPBasinData fbd = fdc.getBasinData(huc); synchronized (fdc) {
if (fbd.getBasins().size() > 0) { cacheRecord = new FFMPAggregateRecord();
// times for Guidance basins will be null
cacheRecord.setTimes(fdc.getOrderedTimes());
String tmpFilePath = fileName + "-" + huc for (Entry<String, FFMPBasinData> entry : fdc
+ ".tmp"; .getBasinMap().entrySet()) {
BufferedOutputStream os = null; FFMPBasinData fbd = entry.getValue();
fbd.setCache();
cacheRecord.setBasinData(fbd);
}
}
try { if (cacheRecord.getBasinsMap().size() > 0) {
File file = new File(tmpFilePath);
file.createNewFile();
if (file.canWrite()) { String tmpFilePath = fileName + ".tmp";
os = new BufferedOutputStream( GZIPOutputStream gos = null;
new FileOutputStream(file));
DynamicSerializationManager dsm = DynamicSerializationManager try {
.getManager(SerializationType.Thrift); File file = new File(tmpFilePath);
dsm.serialize(fbd, os); file.createNewFile();
fileNames.put(tmpFilePath, fileName
+ "-" + huc + ".bin"); if (file.canWrite()) {
} else { gos = new GZIPOutputStream(new BufferedOutputStream(
statusHandler new FileOutputStream(file)));
.handle(Priority.WARN, DynamicSerializationManager dsm = DynamicSerializationManager
"Can not write buddy file: " .getManager(SerializationType.Thrift);
+ file.getAbsolutePath()); dsm.serialize(cacheRecord, gos);
} fileNames.put(tmpFilePath, fileName
} catch (SerializationException e) { + ".bin");
statusHandler.handle(Priority.ERROR, } else {
"Serialization Error Writing buddy file: " statusHandler.handle(
+ e.getMessage()); Priority.WARN,
} catch (IOException e) { "Can not write cache file: "
statusHandler.handle(Priority.ERROR, + file.getAbsolutePath());
"IO Error Writing buddy file: " }
+ e.getMessage()); } catch (SerializationException e) {
} catch (Exception e) { statusHandler.handle(Priority.ERROR,
statusHandler.handle(Priority.ERROR, "Serialization Error Writing cache file: "
"General Error Writing buddy file: " + e.getMessage());
+ e.getMessage()); } catch (IOException e) {
} finally { statusHandler.handle(
if (os != null) { Priority.ERROR,
os.close(); "IO Error Writing cache file: "
} + e.getMessage());
} } catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"General Error Writing cache file: "
+ e.getMessage());
} finally {
if (gos != null) {
gos.close();
} }
} }
} }
} }
} catch (Exception e) { } catch (Exception e) {
statusHandler statusHandler.handle(Priority.ERROR,
.handle(Priority.ERROR, "Error writing cache File: " + e.getMessage());
"Error writing Buddy File group: "
+ e.getMessage());
} finally { } finally {
// rename the files to real path // rename the files to real path
try { try {
@ -1684,31 +1708,29 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle( statusHandler.handle(
Priority.ERROR, Priority.ERROR,
"IO Error Renaming buddy file: " "IO Error Renaming cache file: "
+ e.getMessage()); + e.getMessage());
} }
} }
} catch (Exception e) { } catch (Exception e) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"IO Error writing buddy files: " + e.getMessage()); "IO Error writing cache files: " + e.getMessage());
} }
} }
} }
/** /**
* Load existing buddy file * Load existing cache file
* *
* @param sourceSiteDataKey * @param sourceSiteDataKey
* @param huc
* @param wfo * @param wfo
* @return * @return
*/ */
public boolean checkBuddyFile(String sourceSiteDataKey, String huc, public boolean checkCacheFile(String sourceSiteDataKey, String wfo,
String wfo, Date backDate) { Date backDate) {
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + "-" File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + ".bin");
+ huc + ".bin");
String sourceName = sourceSiteDataKey.split("-")[0]; String sourceName = sourceSiteDataKey.split("-")[0];
SourceXML source = getSourceConfig().getSourceByDisplayName(sourceName); SourceXML source = getSourceConfig().getSourceByDisplayName(sourceName);
@ -1875,8 +1897,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteFFTIFileName(fftiName)); getAbsoluteFFTIFileName(fftiName));
try { try {
ffti = (FFTIData) SerializationUtil.transformFromThrift(FileUtil ffti = SerializationUtil.transformFromThrift(FFTIData.class,
.file2bytes(f.getFile(), true)); FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) { } catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR, statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName(), fnfe); "Unable to locate file " + f.getName(), fnfe);

6
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPConfig.java
View file

@ -221,12 +221,12 @@ public class FFMPConfig {
} }
/** /**
* Get the DHR record * Get the RADAR record
* *
* @param uri * @param uri
* @return * @return
*/ */
private Object getDHRRecord(String uri) { private Object getRADARRecord(String uri) {
Object record = null; Object record = null;
try { try {
record = ScanCommonUtils.getRadarRecord(uri); record = ScanCommonUtils.getRadarRecord(uri);
@ -398,7 +398,7 @@ public class FFMPConfig {
} else if (source.getDataType().equals( } else if (source.getDataType().equals(
FFMPSourceConfigurationManager.DATA_TYPE.RADAR FFMPSourceConfigurationManager.DATA_TYPE.RADAR
.getDataType())) { .getDataType())) {
Object dataObject = getDHRRecord(dataUri); Object dataObject = getRADARRecord(dataUri);
if (dataObject != null) { if (dataObject != null) {
// process as a VGB too // process as a VGB too

12
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java
View file

@ -22,6 +22,7 @@ package com.raytheon.uf.edex.plugin.ffmp.common;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Date; import java.util.Date;
import java.util.List;
import java.util.Map.Entry; import java.util.Map.Entry;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin; import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
@ -30,6 +31,7 @@ import com.raytheon.uf.common.dataplugin.ffmp.FFMPDataContainer;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin; import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord; import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.monitor.xml.SourceXML; import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator; import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
/** /**
@ -41,6 +43,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 29 July, 2011 6772 dhladky Initial creation * 29 July, 2011 6772 dhladky Initial creation
* 29 July, 2012 578 dhladky memory work * 29 July, 2012 578 dhladky memory work
* 27 Jan, 2013 1478 dhladky Changed arraylist to list for times, more constants
* </pre> * </pre>
* *
* @author dhladky * @author dhladky
@ -96,15 +99,16 @@ public class FFMPInterpolatedGuidanceDelay {
qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName() qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName()
+ "-" + siteKey + "-" + siteKey, hucs, backDate); + "-" + siteKey + "-" + siteKey, hucs, backDate);
long expirationTime = qpeSource.getExpirationMinutes(siteKey) * 60 * 1000; long expirationTime = qpeSource.getExpirationMinutes(siteKey)
// determine lag_time * TimeUtil.MILLIS_PER_MINUTE;
// determine lag_time
long lagTime = (currentRecord.getDataTime().getRefTime().getTime()) long lagTime = (currentRecord.getDataTime().getRefTime().getTime())
+ (long) (ffgSource.getDurationHour() * 60 * 1000); + (long) (ffgSource.getDurationHour() * TimeUtil.MILLIS_PER_MINUTE);
// Determine hour fraction. // Determine hour fraction.
int fraction_Hr = (int) (((float) (currentRecord.getDataTime() int fraction_Hr = (int) (((float) (currentRecord.getDataTime()
.getRefTime().getTime() - (lagTime - guidFrequency))) / (float) guidFrequency); .getRefTime().getTime() - (lagTime - guidFrequency))) / (float) guidFrequency);
// Gets the ordered times for QPE // Gets the ordered times for QPE
ArrayList<Date> orderedTimes = qpeContainer List<Date> orderedTimes = qpeContainer
.getOrderedTimes(currentRecord.getDataTime().getRefTime()); .getOrderedTimes(currentRecord.getDataTime().getRefTime());
// EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld) // EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld)

33
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTI.java
View file

@ -44,6 +44,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator; import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
/** /**
@ -54,8 +55,8 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* SOFTWARE HISTORY * SOFTWARE HISTORY
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Apr 01, 2011 dhladky Initial creation * Apr 01, 2011 dhladky Initial creation
* July 13, 2012 dhladky Revamped to help memory * July 13, 2012 dhladky Revamped to help memory
* *
* </pre> * </pre>
* *
@ -443,7 +444,6 @@ public class FFTI implements Runnable {
"Failed to evaluate Ratio/Diff. " "Failed to evaluate Ratio/Diff. "
+ attribute.getAttributeName() + ": " + displayName + attribute.getAttributeName() + ": " + displayName
+ "\n" + e); + "\n" + e);
e.printStackTrace();
} }
} }
@ -826,9 +826,9 @@ public class FFTI implements Runnable {
} }
long cur = config.getDate().getTime(); long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000); long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack); Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * 60 * 1000; long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
FFMPDataContainer fdc = null; FFMPDataContainer fdc = null;
@ -894,6 +894,15 @@ public class FFTI implements Runnable {
return accumulator; return accumulator;
} }
/**
* Gets the ratio and difference values for this site
* @param qSourceKey
* @param qSiteKey
* @param ffgType
* @param duration
* @param unit
* @return
*/
private FFTIRatioDiff getRatioAndDiffForSite(String qSourceKey, private FFTIRatioDiff getRatioAndDiffForSite(String qSourceKey,
String qSiteKey, String ffgType, double duration, String unit) { String qSiteKey, String ffgType, double duration, String unit) {
@ -923,13 +932,13 @@ public class FFTI implements Runnable {
values.setUnit(unit); values.setUnit(unit);
long cur = config.getDate().getTime(); long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000); long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack); Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * 60 * 1000; long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
// make sure we have data // make sure we have data
Date ffgBackDate = new Date(config.getDate().getTime() Date ffgBackDate = new Date(config.getDate().getTime()
- (3600 * 1000 * 24)); - (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
String primarySource = ffmpgen.fscm.getPrimarySource(ffmpQSource); String primarySource = ffmpgen.fscm.getPrimarySource(ffmpQSource);
ProductXML product = ffmpgen.fscm.getProduct(primarySource); ProductXML product = ffmpgen.fscm.getProduct(primarySource);
@ -950,7 +959,7 @@ public class FFTI implements Runnable {
if (guidSourceExpiration == 0l) { if (guidSourceExpiration == 0l) {
guidSourceExpiration = iguidSource guidSourceExpiration = iguidSource
.getExpirationMinutes(qSiteKey) * 60 * 1000; .getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
break; break;
} }
} }
@ -1046,7 +1055,7 @@ public class FFTI implements Runnable {
SourceXML ffmpQSource, double duration, String qSiteKey) { SourceXML ffmpQSource, double duration, String qSiteKey) {
long cur = config.getDate().getTime(); long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000); long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack); Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey); long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey);
Double gapVal = 0.0; Double gapVal = 0.0;
@ -1054,14 +1063,14 @@ public class FFTI implements Runnable {
if (qpeContainer.getOrderedTimes(backDate) != null) { if (qpeContainer.getOrderedTimes(backDate) != null) {
gapVal = 0.0; gapVal = 0.0;
ArrayList<FFMPGap> gaps = FFMPGap.getGaps( List<FFMPGap> gaps = FFMPGap.getGaps(
qpeContainer.getOrderedTimes(backDate), expirationTime, qpeContainer.getOrderedTimes(backDate), expirationTime,
backDate, config.getDate()); backDate, config.getDate());
for (FFMPGap gap : gaps) { for (FFMPGap gap : gaps) {
gapVal += gap.getGap(); gapVal += gap.getGap();
} }
gapVal = gapVal / 60; gapVal = gapVal / TimeUtil.MINUTES_PER_HOUR;
} }
return gapVal; return gapVal;