xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

13.1.2-17 baseline

Browse source 
Former-commit-id: e2358ae19b [formerly 4ae6bd9716] [formerly 4855183325] [formerly e2358ae19b [formerly 4ae6bd9716] [formerly 4855183325] [formerly 01c544953e [formerly 4855183325 [formerly 190f96b29b2c3c9e61150f3789e9fa86e2709e8f]]]]
Former-commit-id: 01c544953e
Former-commit-id: 01f31354d6 [formerly 8c3799c78e] [formerly 557298a18681895f206a6a9e732ef82d906f1b48 [formerly 8ceb80ebf2]]
Former-commit-id: 901b6de36c7b894750ea9c9124941e59f5b85206 [formerly 64de3ed540]
Former-commit-id: 3124695f8a
This commit is contained in:
Steve Harris 2013-02-14 10:12:02 -05:00
parent b6e0c03674
commit eedd6255cd
25 changed files with 946 additions and 542 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

98
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java
View file

@ -28,6 +28,7 @@ import org.eclipse.ui.PlatformUI;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPCacheRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceInterpolation;
@ -87,6 +88,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 04/03/10 4494 D. Hladky Initial release
* 01/27/13 1478 D. Hladky revamped cache file format, removed duplicate times
*
* </pre>
*
@ -452,28 +454,21 @@ public class FFMPMonitor extends ResourceMonitor {
* @param siteKey
* @param dataKey
* @param source
* @param huc
*/
public void insertFFMPData(FFMPBasinData data, String siteKey,
String source, String huc) {
public void insertFFMPData(FFMPAggregateRecord data, String siteKey,
String source) {
final String fsiteKey = siteKey;
final FFMPBasinData fdata = data;
final String fsource = source;
final String fhuc = huc;
VizApp.runAsync(new Runnable() {
@Override
public void run() {
if (ffmpData.containsKey(fsiteKey)) {
if (ffmpData.get(fsiteKey).containsKey(fsource)) {
ffmpData.get(fsiteKey).get(fsource)
.setBasinBuddyData(fdata, fhuc);
}
if (ffmpData.containsKey(siteKey)) {
if (ffmpData.get(siteKey).containsKey(source)) {
for (Entry<String, FFMPBasinData> entry : data.getBasinsMap()
.entrySet()) {
FFMPBasinData basinData = entry.getValue();
basinData.populate(data.getTimes());
ffmpData.get(siteKey).get(source)
.setCacheData(basinData, basinData.getHucLevel());
}
}
});
}
}
/**
@ -507,52 +502,35 @@ public class FFMPMonitor extends ResourceMonitor {
public void populateFFMPBasin(String dataUri, String siteKey,
String source, String phuc, FFMPBasin basin) throws VizException {
final String fdataUri = dataUri;
final String fsiteKey = siteKey;
final String fhuc = phuc;
final String fsource = source;
final FFMPBasin fbasin = basin;
if (dataUri != null) {
ConcurrentMap<String, String> uris = getUriMap(siteKey, source,
phuc);
if (!uris.containsKey(dataUri)) {
try {
SourceXML sourceXML = fscm.getSource(source);
FFMPCacheRecord ffmpRec = populateFFMPRecord(true, dataUri,
siteKey, source, phuc);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory.getDataStore(loc);
VizApp.runAsync(new Runnable() {
@Override
public void run() {
if (fdataUri != null) {
ConcurrentMap<String, String> uris = getUriMap(fsiteKey,
fsource, fhuc);
if (!uris.containsKey(fdataUri)) {
try {
SourceXML sourceXML = fscm.getSource(fsource);
FFMPCacheRecord ffmpRec = populateFFMPRecord(true,
fdataUri, fsiteKey, fsource, fhuc);
//FFMPRecord ffmpRec = loadRecordFromDatabase(fdataUri);
File loc = HDF5Util.findHDF5Location(ffmpRec);
IDataStore dataStore = DataStoreFactory
.getDataStore(loc);
if (sourceXML.getSourceType().equals(
SOURCE_TYPE.GAGE.getSourceType())
&& fhuc.equals("ALL")) {
ffmpRec.retrieveVirtualBasinFromDataStore(
dataStore, fdataUri,
getTemplates(fsiteKey), ffmpRec
.getDataTime().getRefTime(),
fbasin);
} else {
ffmpRec.retrieveBasinFromDataStore(dataStore,
fdataUri, getTemplates(fsiteKey), fhuc,
ffmpRec.getDataTime().getRefTime(),
ffmpRec.getSourceName(), fbasin);
}
} catch (Throwable e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, "
+ fdataUri, e);
}
if (sourceXML.getSourceType().equals(
SOURCE_TYPE.GAGE.getSourceType())
&& phuc.equals("ALL")) {
ffmpRec.retrieveVirtualBasinFromDataStore(dataStore,
dataUri, getTemplates(siteKey), ffmpRec
.getDataTime().getRefTime(), basin);
} else {
ffmpRec.retrieveBasinFromDataStore(dataStore, dataUri,
getTemplates(siteKey), phuc, ffmpRec
.getDataTime().getRefTime(), ffmpRec
.getSourceName(), basin);
}
} catch (Throwable e) {
statusHandler.handle(Priority.PROBLEM,
"FFMP Can't retrieve FFMP URI, " + dataUri, e);
}
}
});
}
}
/**

335
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataLoader.java
View file

@ -29,8 +29,9 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableMap;
import java.util.zip.GZIPInputStream;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.monitor.config.FFMPRunConfigurationManager;
import com.raytheon.uf.common.monitor.xml.FFMPRunXML;
import com.raytheon.uf.common.monitor.xml.ProductRunXML;
@ -40,6 +41,10 @@ import com.raytheon.uf.common.ohd.AppsDefaults;
import com.raytheon.uf.common.serialization.DynamicSerializationManager;
import com.raytheon.uf.common.serialization.DynamicSerializationManager.SerializationType;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.viz.core.VizApp;
import com.raytheon.uf.viz.monitor.ffmp.FFMPMonitor;
import com.raytheon.uf.viz.monitor.ffmp.ui.dialogs.FFMPConfig;
@ -57,6 +62,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent;
* ------------ ---------- ----------- --------------------------
* 28 Feb, 2011 7587 dhladky Initial creation
* 25 Jan, 2012 DR13839 gzhang Handle Uris and Huc processing
* 01/27/13 1478 D. Hladky revamped the cache file format to help NAS overloading
* </pre>
*
* @author dhladky
@ -64,8 +70,7 @@ import com.raytheon.uf.viz.monitor.ffmp.ui.listeners.FFMPLoaderEvent;
*/
public class FFMPDataLoader extends Thread {
// private static final transient IUFStatusHandler statusHandler = UFStatus
// .getHandler(FFMPDataLoader.class);
private static final IUFStatusHandler statusHandler = UFStatus.getHandler(FFMPDataLoader.class);
private String sharePath = null;
@ -154,7 +159,7 @@ public class FFMPDataLoader extends Thread {
ProductRunXML productRun = runner.getProduct(siteKey);
ArrayList<String> qpfSources = new ArrayList<String>();
String layer = config.getFFMPConfigData().getLayer();
boolean isProductLoad = true;
String rateURI = null;
@ -225,179 +230,159 @@ public class FFMPDataLoader extends Thread {
hucsToLoad.add("ALL");
}
for (String phuc : hucsToLoad) {
if (isDone) {
return;
}
if (isDone) {
return;
// rate
if (rateURI != null) {
fireLoaderEvent(loadType, "Processing " + product.getRate(),
isDone);
for (String phuc : hucsToLoad) {
getMonitor().processUri(isProductLoad, rateURI, siteKey,
product.getRate(), timeBack, phuc);
}
fireLoaderEvent(loadType, product.getRate(), isDone);
}
if (phuc.equals("VIRTUAL")) {
// we don't do the virtuals like this
continue;
} else {
// rate
if (rateURI != null) {
fireLoaderEvent(loadType,
"Processing " + product.getRate() + "/" + phuc,
isDone);
// qpes
fireLoaderEvent(loadType, "Processing " + product.getQpe(), isDone);
FFMPAggregateRecord qpeCache = null;
getMonitor().processUri(isProductLoad, rateURI,
siteKey, product.getRate(), timeBack, phuc);
fireLoaderEvent(loadType, product.getRate() + "/"
+ phuc, isDone);
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig().getSource(
product.getQpe());
qpeCache = readCacheFile(source, dataKey, wfo);
if (qpeCache != null) {
getMonitor().insertFFMPData(qpeCache, siteKey,
product.getQpe());
}
}
// Use this method of QPE data retrieval if you don't have cache files
if (!qpeURIs.isEmpty() && qpeCache == null) {
for (String phuc : hucsToLoad) {
if (phuc.equals(layer)
|| phuc.equals("ALL")) {
getMonitor().processUris(qpeURIs, isProductLoad,
siteKey, product.getQpe(), timeBack, phuc);
}
}
}
// qpes
fireLoaderEvent(loadType, "Processing " + product.getQpe()
+ "/" + phuc, isDone);
FFMPBasinData qpeData = null;
fireLoaderEvent(loadType, product.getQpe(), isDone);
if (loadType == LOADER_TYPE.INITIAL) {
int i = 0;
for (NavigableMap<Date, List<String>> qpfURIs : qpfs) {
// qpf
fireLoaderEvent(loadType, "Processing " + product.getQpf(i),
isDone);
FFMPAggregateRecord qpfCache = null;
SourceXML source = getMonitor().getSourceConfig()
.getSource(product.getQpe());
if (loadType == LOADER_TYPE.INITIAL) {
qpeData = readLoaderBuddyFile(source, phuc, dataKey,
wfo);
SourceXML source = getMonitor().getSourceConfig()
.getSource(qpfSources.get(i));
if (qpeData != null) {
String pdataKey = findQPFHomeDataKey(source);
qpfCache = readCacheFile(source, pdataKey, wfo);
getMonitor().insertFFMPData(qpeData, siteKey,
product.getQpe(), phuc);
}
}
if (qpfCache != null) {
for (String phuc : hucsToLoad) {
if ((phuc.equals(layer) || phuc.equals("ALL"))
&& loadType == LOADER_TYPE.INITIAL
&& source.getSourceName().equals(
config.getFFMPConfigData()
.getIncludedQPF())) {
if (!qpfURIs.isEmpty()) {
if (!qpeURIs.isEmpty() && qpeData == null) {
if (phuc.equals(config.getFFMPConfigData().getLayer())
|| phuc.equals("ALL")) {
getMonitor().processUris(qpeURIs, isProductLoad,
siteKey, product.getQpe(), timeBack, phuc);
}
}
fireLoaderEvent(loadType, product.getQpe() + "/" + phuc,
isDone);
int i = 0;
for (NavigableMap<Date, List<String>> qpfURIs : qpfs) {
// qpf
fireLoaderEvent(loadType,
"Processing " + product.getQpf(i) + "/" + phuc,
isDone);
FFMPBasinData qpfData = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig()
.getSource(qpfSources.get(i));
String pdataKey = findQPFHomeDataKey(source);
qpfData = readLoaderBuddyFile(source, phuc,
pdataKey, wfo);
if (qpfData != null) {
if ((phuc.equals(config.getFFMPConfigData()
.getLayer()) || phuc.equals("ALL"))
&& loadType == LOADER_TYPE.INITIAL
&& source.getSourceName().equals(
config.getFFMPConfigData()
.getIncludedQPF())) {
if (!qpfURIs.isEmpty()) {
getMonitor().processUris(qpfURIs,
isProductLoad, siteKey,
source.getSourceName(),
timeBack, phuc);
}
getMonitor().processUris(qpfURIs,
isProductLoad, siteKey,
source.getSourceName(), timeBack,
phuc);
}
getMonitor().insertFFMPData(qpfData, siteKey,
source.getSourceName(), phuc);
}
}
// if (isUrisProcessNeeded(qpfData,qpfURIs))
// {/*DR13839*/
if ((qpfData == null) && !qpfURIs.isEmpty()) {
if (phuc.equals(config.getFFMPConfigData()
.getLayer()) || phuc.equals("ALL")) { // old
// code:
// keep
// for
// reference*/
// if (isHucProcessNeeded(phuc)) {/*DR13839*/
getMonitor().processUris(qpfURIs,
isProductLoad, siteKey,
product.getQpf(i), timeBack, phuc);
}
}
fireLoaderEvent(loadType, product.getQpf(i) + "/"
+ phuc, isDone);
i++;
getMonitor().insertFFMPData(qpfCache, siteKey,
source.getSourceName());
}
}
// virtuals only have data for ALL
if (phuc.equals("ALL")) {
// if (isUrisProcessNeeded(qpfData,qpfURIs))
// {/*DR13839*/
// Use this method of QPF data retrieval if you don't have cache files
if ((qpfCache == null) && !qpfURIs.isEmpty()) {
for (String phuc : hucsToLoad) {
if (phuc.equals(layer)
|| phuc.equals("ALL")) { // old
// code:
// keep
// for
// reference*/
// if (isHucProcessNeeded(phuc)) {/*DR13839*/
getMonitor().processUris(qpfURIs, isProductLoad,
siteKey, product.getQpf(i), timeBack, phuc);
}
}
}
fireLoaderEvent(loadType, product.getQpf(i), isDone);
i++;
}
fireLoaderEvent(loadType, "Processing " + product.getVirtual(),
isDone);
FFMPAggregateRecord vgbCache = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig().getSource(
product.getVirtual());
vgbCache = readCacheFile(source, dataKey, wfo);
if (vgbCache != null) {
getMonitor().insertFFMPData(vgbCache, siteKey,
product.getVirtual());
}
}
// Use this method of Virtual data retrieval if you don't have cache files
if ((vgbCache == null) && !virtualURIs.isEmpty()) {
getMonitor().processUris(virtualURIs, isProductLoad, siteKey,
product.getVirtual(), timeBack, "ALL");
}
fireLoaderEvent(loadType, product.getVirtual(), isDone);
// process guidance all for all only, never uses cache files
for (String type : productRun.getGuidanceTypes(product)) {
ArrayList<SourceXML> guidSources = productRun
.getGuidanceSources(product, type);
for (SourceXML guidSource : guidSources) {
NavigableMap<Date, List<String>> iguidURIs = guids
.get(guidSource.getSourceName());
fireLoaderEvent(loadType,
"Processing " + product.getVirtual() + "/" + phuc,
"Processing " + guidSource.getSourceName(), isDone);
getMonitor().processUris(iguidURIs, isProductLoad, siteKey,
guidSource.getSourceName(), timeBack, "ALL");
fireLoaderEvent(loadType, guidSource.getSourceName(),
isDone);
FFMPBasinData vgbData = null;
if (loadType == LOADER_TYPE.INITIAL) {
SourceXML source = getMonitor().getSourceConfig()
.getSource(product.getVirtual());
vgbData = readLoaderBuddyFile(source, phuc, dataKey,
wfo);
if (vgbData != null) {
getMonitor().insertFFMPData(vgbData, siteKey,
product.getVirtual(), phuc);
}
}
if ((vgbData == null) && !virtualURIs.isEmpty()) {
getMonitor().processUris(virtualURIs, isProductLoad,
siteKey, product.getVirtual(), timeBack, phuc);
}
fireLoaderEvent(loadType,
product.getVirtual() + "/" + phuc, isDone);
}
// process guidance all at once
for (String type : productRun.getGuidanceTypes(product)) {
ArrayList<SourceXML> guidSources = productRun
.getGuidanceSources(product, type);
for (SourceXML guidSource : guidSources) {
NavigableMap<Date, List<String>> iguidURIs = guids
.get(guidSource.getSourceName());
fireLoaderEvent(loadType,
"Processing " + guidSource.getSourceName()
+ "/" + phuc, isDone);
getMonitor().processUris(iguidURIs, isProductLoad,
siteKey, guidSource.getSourceName(), timeBack,
phuc);
fireLoaderEvent(loadType, guidSource.getSourceName()
+ "/" + phuc, isDone);
}
}
fireLoaderEvent(loadType, phuc + " Load complete", isDone);
}
} catch (Exception e) {
System.err.println("FFMP Data Loader terminated...."
+ e.getMessage());
statusHandler.handle(Priority.PROBLEM,"General Problem in Loading FFMP Data", e);
} finally {
isDone = true;
}
@ -410,7 +395,7 @@ public class FFMPDataLoader extends Thread {
}
long endTime = (System.currentTimeMillis()) - time;
System.out.println("Loader took: " + endTime / 1000 + " seconds");
System.out.println(loadType.loaderType + " Loader took: " + endTime / 1000 + " seconds");
fireLoaderEvent(loadType, message, isDone);
}
@ -467,19 +452,19 @@ public class FFMPDataLoader extends Thread {
};
/**
* Loads the loader buddy files
* Loads the Cache files
*
* @param sourceName
* @param huc
* @param wfo
* @return
*/
private FFMPBasinData readLoaderBuddyFile(SourceXML source, String huc,
String pdataKey, String wfo) {
private FFMPAggregateRecord readCacheFile(SourceXML source, String pdataKey, String wfo) {
long time = System.currentTimeMillis();
String sourceName = source.getSourceName();
File file = new File(sharePath + wfo + File.separator + sourceName
+ "-" + siteKey + "-" + pdataKey + "-" + huc + ".bin");
+ "-" + siteKey + "-" + pdataKey + ".bin");
File lockFile = new File(sharePath + wfo + File.separator + sourceName
+ "-" + siteKey + "-" + pdataKey + ".lock");
@ -489,22 +474,22 @@ public class FFMPDataLoader extends Thread {
sleep(100);
i++;
} catch (InterruptedException e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR,"Took to long to load Cache Record", e);
}
}
break;
}
System.out.println("Buddy File expected path: "
System.out.println("Cache File expected path: "
+ file.getAbsolutePath());
FFMPBasinData basinData = null;
FFMPAggregateRecord cacheRecord = null;
if (file.exists()) {
System.out.println("Last mod: " + new Date(file.lastModified()));
if (file.lastModified() > (System.currentTimeMillis() - (6 * 1000 * 3600))) {
if (file.lastModified() > (System.currentTimeMillis() - (6 * TimeUtil.MILLIS_PER_HOUR))) {
while (lockFile.exists()) {
for (int i = 0; i < 4; i++) {
@ -514,39 +499,42 @@ public class FFMPDataLoader extends Thread {
sleep(100);
i++;
} catch (InterruptedException e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR,"ERROR in Loading Cache Record", e);
}
}
break;
}
BufferedInputStream is = null;
GZIPInputStream gis = null;
try {
System.out.println("Loading file: " + file.getName());
is = new BufferedInputStream(new FileInputStream(file));
gis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)));
DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift);
basinData = (FFMPBasinData) dsm.deserialize(is);
cacheRecord = (FFMPAggregateRecord) dsm.deserialize(gis);
} catch (SerializationException e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR,"Serialization ERROR in Loading Cache Record", e);
} catch (IOException e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR,"IO ERROR in Loading Cache Record", e);
} finally {
if (is != null) {
if (gis != null) {
try {
is.close();
gis.close();
} catch (IOException e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR,"GENRAL ERROR in Loading Cache Record", e);
}
}
}
}
}
long time2 = System.currentTimeMillis();
System.out.println("FFMP Cache file Load took: "+ (time2 - time) + " ms");
return basinData;
return cacheRecord;
}
@ -564,7 +552,7 @@ public class FFMPDataLoader extends Thread {
for (ProductRunXML product : runManager.getProducts()) {
File file = new File(sharePath + wfo + File.separator
+ source.getSourceName() + "-" + siteKey + "-"
+ product.getProductKey() + "-ALL.bin");
+ product.getProductKey() + ".bin");
if (file.exists()) {
return product.getProductKey();
@ -572,7 +560,6 @@ public class FFMPDataLoader extends Thread {
}
return siteKey;
}
}

5
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java
View file

@ -151,7 +151,8 @@ import com.vividsolutions.jts.geom.Point;
* 11 Apr. 2012 DR 14522 gzhang Fixing invalid thread error.
* 31 July 2012 14517 mpduff Fix for blanking map on update.
* 14 Sep 2012 1048 njensen Code cleanup
*
* 10 Jan 2013 1475 dhladky Some cleanup
* 27 Jan 2013 1478 dhladky Changed gap collection to a generic list insted of Arraylist
* </pre>
*
* @author dhladky
@ -3940,7 +3941,7 @@ public class FFMPResource extends
*
* @return Array of Gap data
*/
public ArrayList<FFMPGap> getGaps() {
public List<FFMPGap> getGaps() {
synchronized (timeOrderedKeys) {
return FFMPGap.getGaps(getTimeOrderedKeys(), getResourceData()
.getPrimarySourceXML().getExpirationMinutes(getSiteKey()),

8
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/META-INF/services/com.raytheon.uf.common.serialization.ISerializableObject
View file

@ -1,8 +1,2 @@
com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord
com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPVirtualGageBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin
com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData
com.raytheon.uf.common.dataplugin.ffmp.SourceBinList
com.raytheon.uf.common.dataplugin.ffmp.SourceBin
com.raytheon.uf.common.dataplugin.ffmp.SourceBinEntry

104
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPAggregateRecord.java Normal file
View file

@ -0,0 +1,104 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
/**
* Cache Record implementation for FFMP plugin
* Eventually this will become a full record implementation
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Created to reduce memory and disk read/writes for FFMP
*
* </pre>
*
* @author dhladky
* @version 1
*/
@DynamicSerialize
public class FFMPAggregateRecord implements ISerializableObject {
private static final long serialVersionUID = 76774564363471L;
/**
*
*/
public FFMPAggregateRecord() {
}
@DynamicSerializeElement
private HashMap<String, FFMPBasinData> basinsMap = new HashMap<String, FFMPBasinData>();
@DynamicSerializeElement
private List<Long> times = new ArrayList<Long>();
public void setTimes(List<Long> times) {
this.times = times;
}
public List<Long> getTimes() {
return times;
}
public void setBasinsMap(HashMap<String, FFMPBasinData> basinsMap) {
this.basinsMap = basinsMap;
}
public HashMap<String, FFMPBasinData> getBasinsMap() {
return basinsMap;
}
/**
* Add a basin Data Cache object
* @param cacheData
*/
public void setBasinData(FFMPBasinData cacheData) {
basinsMap.put(cacheData.getHucLevel(), cacheData);
}
/**
* Gets the BasinData object
* @param huc
* @return
*/
public FFMPBasinData getBasinData(String huc) {
if (basinsMap.containsKey(huc)) {
return basinsMap.get(huc);
}
return null;
}
}

87
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasin.java
View file

@ -22,14 +22,11 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.persistence.Transient;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
@ -45,30 +42,49 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 06/22/09 2152 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added support for writing aggregate record cache
*
* </pre>
*
* @author dhladky
* @version 1
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class FFMPBasin implements ISerializableObject, Cloneable {
/** pfafstetter id(key) in GIS **/
@DynamicSerializeElement
@XmlElement
protected Long pfaf;
/** boolean aggregator **/
@DynamicSerializeElement
@XmlAttribute
protected boolean aggregated = false;
@DynamicSerializeElement
@XmlElement
/** object used in calculations
* not serialized
**/
@Transient
protected TreeMap<Date, Float> values;
/** object used for serialization **/
@DynamicSerializeElement
public float[] cacheValues;
/**
* Get the float array of serialized values
* @return
*/
public float[] getCacheValues() {
return cacheValues;
}
/**
* Set the serialized array of cache values
* @param cacheValues
*/
public void setCacheValues(float[] cacheValues) {
this.cacheValues = cacheValues;
}
/**
* @return the pfaf_id
@ -326,14 +342,23 @@ public class FFMPBasin implements ISerializableObject, Cloneable {
}
/**
* No arg hibernate constructor
* No arg serial constructor
*/
public FFMPBasin() {
values = new TreeMap<Date, Float>(new Comparator<Date>() {
@Override
public int compare(Date o1, Date o2) {
// Null checks?
return (int)(o2.getTime() - o1.getTime()) ;
}
});
}
/**
* useful constructor
* Useful constructor
* @param pfaf
* @param aggregated
*/
public FFMPBasin(Long pfaf, boolean aggregated) {
setPfaf(pfaf);
@ -342,11 +367,45 @@ public class FFMPBasin implements ISerializableObject, Cloneable {
@Override
public int compare(Date o1, Date o2) {
// Null checks?
return (int)Math.signum(o2.getTime() - o1.getTime()) ;
return (int)(o2.getTime() - o1.getTime()) ;
}
});
}
/**
* Populates the values from the cache
*
* @param times
*/
public void populate(List<Long> times) {
// safe to avoid Array Index Exceptions / shouldn't happen but.....
if (cacheValues != null && (times.size() == cacheValues.length)) {
int i = 0;
for (Long time : times) {
values.put(new Date(time), cacheValues[i]);
i++;
}
//System.out.println("populated :"+i+" pfaf : "+pfaf);
}
}
/**
* populates the serialized array
*/
public void setCache() {
cacheValues = new float[values.size()];
int i = 0;
for (Date date: values.descendingKeySet()) {
cacheValues[i] = values.get(date);
i++;
}
//System.out.println("wrote :"+i+" pfaf : "+pfaf);
}
/**
* purge out old entries

35
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPBasinData.java
View file

@ -22,11 +22,7 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
@ -42,14 +38,13 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 06/22/09 2152 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added support for write of aggregate record cache
*
* </pre>
*
* @author dhladky
* @version 1
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class FFMPBasinData implements ISerializableObject {
@ -62,11 +57,9 @@ public class FFMPBasinData implements ISerializableObject {
// defaults
@DynamicSerializeElement
@XmlElement
private String hucLevel;
@DynamicSerializeElement
@XmlElement
private HashMap<Long, FFMPBasin> basins = new HashMap<Long, FFMPBasin>();
/**
@ -86,7 +79,7 @@ public class FFMPBasinData implements ISerializableObject {
}
/**
* Used only internally
* get the basin map
*
* @return
*/
@ -95,7 +88,7 @@ public class FFMPBasinData implements ISerializableObject {
}
/**
* DONT USE THIS EVER!!!!!
* Sets the basin map
*
* @param basins
*/
@ -545,5 +538,25 @@ public class FFMPBasinData implements ISerializableObject {
basin.purgeData(date);
}
}
/**
* populates data from the cache
*
* @param times
*/
public void populate(List<Long> times) {
for (FFMPBasin basin : basins.values()) {
basin.populate(times);
}
}
/**
* populates the serialized array/objects
*/
public void setCache() {
for (FFMPBasin basin : basins.values()) {
basin.setCache();
}
}
}

28
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPCacheRecord.java
View file

@ -1,4 +1,23 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.awt.Point;
import java.io.IOException;
@ -210,16 +229,16 @@ public class FFMPCacheRecord extends FFMPRecord {
}
/**
* Buddy File reader
* Cache File reader
*
* @param basins
* @param hucName
*/
public void setBasinBuddyData(FFMPBasinData basins, String hucName) {
public void setCacheData(FFMPBasinData basins, String hucName) {
if (getBasinData(hucName) != null) {
basins = getBasinData(hucName, true);
//System.out.println("Adding pieces Buddy Data: "+hucName+" "+getSourceName());
//System.out.println("Adding Cache Data: "+hucName+" "+getSourceName());
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.getBasins()
@ -442,8 +461,7 @@ public class FFMPCacheRecord extends FFMPRecord {
} catch (Throwable e) {
statusHandler.handle(Priority.PROBLEM,
"ERROR Retrieving Map for URI: " + uri
+ "..." + huc);
e.printStackTrace();
+ "..." + huc, e);
}
}
}

119
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPDataContainer.java
View file

@ -20,9 +20,9 @@
package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@ -33,7 +33,7 @@ import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
/**
* FFTI Data Container
* FFMP Data Container
*
* <pre>
*
@ -43,7 +43,8 @@ import com.raytheon.uf.common.status.UFStatus;
* ------------ ---------- ----------- --------------------------
* 03/31/11 5489 D. Hladky Initial release
* 07/31/12 578 D.Hladky finished it
* 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException
* 09/27/12 DR 15471 G.Zhang Fixed ConcurrentModificationException
* 01/27/13 1478 D. Hladky Re-worked to help with memory size and NAS read write stress
* </pre>
*
* @author dhladky
@ -62,15 +63,24 @@ public class FFMPDataContainer {
private String filePath = null;
public FFMPDataContainer() {
// public unused constructor
}
/**
* Usual constructor
* @param sourceName
*/
public FFMPDataContainer(String sourceName) {
this.sourceName = sourceName;
basinDataMap.put("ALL", new FFMPBasinData("ALL"));
// System.out.println("Creating source: " + sourceName);
}
/**
* special constuctor
* @param sourceName
* @param hucs
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
@ -78,6 +88,21 @@ public class FFMPDataContainer {
basinDataMap.put(huc, new FFMPBasinData(huc));
}
}
/**
* new container first time read in from cache
* @param sourceName
* @param hucs
* @param record
*/
public FFMPDataContainer(String sourceName, ArrayList<String> hucs, FFMPAggregateRecord record) {
// System.out.println("Creating source with hucs: " + sourceName);
this.sourceName = sourceName;
for (String huc : hucs) {
FFMPBasinData basinData = record.getBasinData(huc);
basinData.populate(record.getTimes());
basinDataMap.put(huc, basinData);
}
}
/**
* Adds to the cache
@ -400,7 +425,7 @@ public class FFMPDataContainer {
* @param barrierTime
* @return
*/
public ArrayList<Date> getOrderedTimes(Date barrierTime) {
public List<Date> getOrderedTimes(Date barrierTime) {
ArrayList<Date> orderedTimes = new ArrayList<Date>();
try {
HashMap<Long, FFMPBasin> basins = getBasinData("ALL").getBasins();
@ -414,7 +439,32 @@ public class FFMPDataContainer {
}
}
Collections.reverse(orderedTimes);
return orderedTimes;
}
}
} catch (Exception e) {
statusHandler.debug("No ordered times available..."
+ getSourceName());
}
return null;
}
/**
* Gets the list of times for serialization
* @return
*/
public List<Long> getOrderedTimes() {
ArrayList<Long> orderedTimes = new ArrayList<Long>();
try {
HashMap<Long, FFMPBasin> basins = getBasinData("ALL").getBasins();
synchronized (basins) {
for (Entry<Long, FFMPBasin> entry : basins.entrySet()) {
FFMPBasin basin = entry.getValue();
for (Date time : basin.getValues().descendingKeySet()) {
orderedTimes.add(time.getTime());
}
return orderedTimes;
}
@ -422,18 +472,22 @@ public class FFMPDataContainer {
} catch (Exception e) {
statusHandler.debug("No ordered times available..."
+ getSourceName());
return null;
}
return null;
}
/**
* Gets the source name for this Data Container
* @return
*/
public String getSourceName() {
return sourceName;
}
/*
* clean up old junk
/**
* Clean up old junk
* @param backDate
*/
public void purge(Date backDate) {
for (String huc : basinDataMap.keySet()) {
@ -442,27 +496,33 @@ public class FFMPDataContainer {
}
/**
* maybe this will work
* Sets the Cache data for this container
*
* @param basins
* @param hucName
* @param cacheRecord
*/
public void setBasinBuddyData(FFMPBasinData basins, String hucName) {
public void setCacheData(FFMPAggregateRecord cacheRecord) {
for (Entry<Long, FFMPBasin> entry : basins.getBasins().entrySet()) {
FFMPBasin basin = getBasinData(hucName).get(entry.getKey());
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
// create a record from the cache record
FFMPRecord record = new FFMPRecord(cacheRecord);
for (Entry<String, FFMPBasinData> dentry : record.getBasinsMap()
.entrySet()) {
for (Entry<Long, FFMPBasin> entry : dentry.getValue().getBasins()
.entrySet()) {
FFMPBasin basin = entry.getValue();
if (basin != null) {
if (basin instanceof FFMPGuidanceBasin) {
FFMPGuidanceBasin gbasin = (FFMPGuidanceBasin) basin;
gbasin.getGuidValues().putAll(
((FFMPGuidanceBasin) entry.getValue())
.getGuidValues());
} else {
basin.getValues().putAll(entry.getValue().getValues());
}
} else {
basin.getValues().putAll(entry.getValue().getValues());
syncPut(getBasinData(dentry.getKey()), entry.getKey(),
entry.getValue());
}
} else {
syncPut(getBasinData(hucName), entry.getKey(), entry.getValue());
//getBasinData(hucName).put(entry.getKey(), entry.getValue());
}
}
}
@ -523,4 +583,13 @@ public class FFMPDataContainer {
basins.put(key, value);
}
}
/**
* Gets the basin data map
* @return
*/
public ConcurrentHashMap<String, FFMPBasinData> getBasinMap() {
return basinDataMap;
}
}

39
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGap.java
View file

@ -1,4 +1,23 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
/**
* gap for FFMP
@ -10,6 +29,7 @@ package com.raytheon.uf.common.dataplugin.ffmp;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 03/03/11 7334 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Added use of constants for calculations
*
* </pre>
*
@ -19,6 +39,9 @@ package com.raytheon.uf.common.dataplugin.ffmp;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import com.raytheon.uf.common.time.util.TimeUtil;
public class FFMPGap {
@ -62,21 +85,25 @@ public class FFMPGap {
}
/**
* Gets the GAP calculation for an FFMP source
* Get the gaps in the FFMP data
*
* @return Array of Gap data
* @param times
* @param expirationTime
* @param barrierTime
* @param mostRecentTime
* @return
*/
public static ArrayList<FFMPGap> getGaps(ArrayList<Date> times,
public static List<FFMPGap> getGaps(List<Date> times,
long expirationTime, Date barrierTime, Date mostRecentTime) {
ArrayList<FFMPGap> gaps = new ArrayList<FFMPGap>();
long gapStep = expirationTime * 60 * 1000;
long gapStep = expirationTime * TimeUtil.MILLIS_PER_MINUTE;
Date prevTime = null;
// System.out.println("Calling getGaps()...Recent Time: " + mostRecentTime
// + " BarrierTime: " + barrierTime);
if (times.size() == 1) {
FFMPGap gap = new FFMPGap();
long totalMillis = mostRecentTime.getTime() - barrierTime.getTime() - gapStep;
float gapMinutes = (totalMillis)/(60 * 1000);
float gapMinutes = totalMillis/TimeUtil.MILLIS_PER_MINUTE;
if (gapMinutes < 0.0) {
gapMinutes = 0.0f;
}
@ -100,7 +127,7 @@ public class FFMPGap {
FFMPGap gap = new FFMPGap(prevTime, time);
// convert to minutes and set gap
// Need to subtract the expirationTime from the gapTime as well
gap.setGap((gapTime - gapStep) / (60 * 1000));
gap.setGap((gapTime - gapStep) / TimeUtil.MILLIS_PER_MINUTE);
gaps.add(gap);
}
prevTime = time;

27
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGuidanceBasin.java
View file

@ -4,13 +4,10 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.TreeMap;
import javax.persistence.Transient;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.monitor.config.FFFGDataMgr;
import com.raytheon.uf.common.serialization.ISerializableObject;
@ -27,22 +24,21 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 08/22/10 3437 D. Hladky Initial release
* 01/17/13 1478 D. Hladky Removed un-needed XML attributes
*
* </pre>
*
* @author dhladky
* @version 1
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject {
public FFMPGuidanceBasin() {
}
@DynamicSerializeElement
@XmlElement
protected TreeMap<Date, HashMap<String, Float>> guidValues;
@Transient
@ -411,7 +407,10 @@ public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject
}
/**
* useful constructor
* Constructor used in producing a new GuidanceBasin
*
* @param pfaf
* @param aggregated
*/
public FFMPGuidanceBasin(Long pfaf, boolean aggregated) {
setPfaf(pfaf);
@ -457,5 +456,15 @@ public class FFMPGuidanceBasin extends FFMPBasin implements ISerializableObject
}
return buff.toString();
}
public void populate(List<Long> times) {
// does nothing here, don't need to populate anything.
}
/**
* populates the serialized array
*/
public void setCache() {
// does nothing here, this class is serialized as is.
}
}

4
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPGuidanceInterpolation.java
View file

@ -30,6 +30,7 @@ import com.raytheon.uf.common.monitor.config.FFMPSourceConfigurationManager;
import com.raytheon.uf.common.monitor.xml.ProductRunXML;
import com.raytheon.uf.common.monitor.xml.ProductXML;
import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
* Guidance Interpolation
@ -39,6 +40,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 29 Jan, 2010 3915 dhladky Initial creation
* 01/27/13 1478 dhladky Added use of constants
* </pre>
*
* @author dhladky
@ -258,7 +260,7 @@ public class FFMPGuidanceInterpolation {
getSource(orderedHours.get(index)),
null,
manager.getSource(source).getExpirationMinutes(
siteKey) * 60 * 1000);
siteKey) * TimeUtil.MILLIS_PER_MINUTE);
if (dman.isExpired() == false) {
thisVal = dman.adjustValue(thisVal,

36
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPRecord.java
View file

@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map.Entry;
import javax.persistence.Column;
@ -66,6 +67,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 06/03/09 2521 D. Hladky Initial release
* 01/27/13 1478 D. Hladky OUN memory help
*
* </pre>
*
@ -758,5 +760,39 @@ public class FFMPRecord extends ServerSpecificPersistablePluginDataObject
public String getSiteKey() {
return siteKey;
}
/**
* Get the fully cache ready object
* @param fileName
* @return
*/
public FFMPAggregateRecord getCacheRecord() {
FFMPAggregateRecord fdcr = new FFMPAggregateRecord();
for (Entry<String,FFMPBasinData> entry: basinsMap.entrySet()) {
fdcr.setBasinData(entry.getValue());
}
return fdcr;
}
/**
* Creates and populates a version of this record from a cache record
*
* @param fdcr
*/
public FFMPRecord(FFMPAggregateRecord fdcr) {
List<Long> times = fdcr.getTimes();
for (Entry<String, FFMPBasinData> entry : fdcr.getBasinsMap()
.entrySet()) {
FFMPBasinData fbd = entry.getValue();
// Keep in mind times can be null, Guidance basins are like that
fbd.populate(times);
setBasinData(fbd, fbd.getHucLevel());
}
}
}

26
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPVirtualGageBasin.java
View file

@ -1,13 +1,28 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.Comparator;
import java.util.Date;
import java.util.TreeMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -21,14 +36,13 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 02may10 3937 dhladky Setup
* 01/27/13 1478 dhladky Removed un-needed XML annotations
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class FFMPVirtualGageBasin extends FFMPBasin implements
ISerializableObject {

31
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBin.java
View file

@ -1,12 +1,26 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -22,28 +36,25 @@ import com.vividsolutions.jts.geom.Coordinate;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 10/22/10 6581 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
*
* </pre>
*
* @author dhladky
* @version 1
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class SourceBin implements ISerializableObject {
/** sourceName and dataKey **/
@DynamicSerializeElement
@XmlElement
public double[] lats;
@DynamicSerializeElement
@XmlElement
public double[] lons;
@DynamicSerializeElement
@XmlElement
public double[] areaPercent;
public SourceBin() {

43
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBinEntry.java
View file

@ -1,24 +1,53 @@
package com.raytheon.uf.common.dataplugin.ffmp;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.vividsolutions.jts.geom.Coordinate;
/**
* FFMP source binning entry object
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
*
* </pre>
*
* @author dhladky
* @version 1
*/
@DynamicSerialize
@XmlAccessorType(XmlAccessType.NONE)
public class SourceBinEntry {
/** sourceName and dataKey **/
@DynamicSerializeElement
@XmlElement
public Coordinate coor;
@DynamicSerializeElement
@XmlElement
public double area;
public Coordinate getCoor() {

29
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/SourceBinList.java
View file

@ -1,12 +1,26 @@
package com.raytheon.uf.common.dataplugin.ffmp;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.HashMap;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
@ -22,24 +36,21 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 10/22/10 6581 D. Hladky Initial release
* 01/27/13 1478 D. Hladky Removed un needed XML annotations
*
* </pre>
*
* @author dhladky
* @version 1
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class SourceBinList implements ISerializableObject {
/** sourceName and dataKey **/
@DynamicSerializeElement
@XmlElement
public String sourceId;
@DynamicSerializeElement
@XmlElement
public HashMap<Long, SourceBin> sourceMap;
public SourceBinList() {

4
edexOsgi/com.raytheon.uf.common.time/src/com/raytheon/uf/common/time/util/TimeUtil.java
View file

@ -24,6 +24,7 @@ import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -101,6 +102,9 @@ public class TimeUtil {
public static final long MILLIS_PER_YEAR = 3600 * 24 * 1000 * 365;
public static final int SECONDS_PER_MINUTE = 60;
public static final long MINUTES_PER_HOUR = 60;
// create instance of simple date format on class load, as instantiating it
// is expensive the SimpleDateFormat class is not thread-safe,

4
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml
View file

@ -14,8 +14,8 @@
</bean>
<bean id="ffmpProcessThreadPool"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="3" />
<property name="maxPoolSize" value="5" />
<property name="corePoolSize" value="2" />
<property name="maxPoolSize" value="4" />
<property name="keepAliveSeconds" value="60000" />
</bean>

294
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -31,15 +31,19 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executor;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import com.raytheon.edex.msg.DataURINotificationMessage;
import com.raytheon.edex.plugin.radar.dao.RadarStationDao;
import com.raytheon.edex.urifilter.URIFilter;
import com.raytheon.edex.urifilter.URIGenerateMessage;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPAggregateRecord;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasinData;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPDataContainer;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
@ -79,6 +83,7 @@ import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.props.PropertiesFactory;
@ -105,6 +110,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIProcessor;
* 06/21/2009 2521 dhladky Initial Creation.
* 02/03/2011 6500 cjeanbap Fixed NullPointerException.
* 07/31/2011 578 dhladky FFTI modifications
* 01/27/13 1478 D. Hladky Added creation of full cache records to help read write stress on NAS
* </pre>
*
* @author dhladky
@ -134,6 +140,18 @@ public class FFMPGenerator extends CompositeProductGenerator implements
private static final String productType = "ffmp";
/**
* The thought was this will eventually be dynamic when We start writing
* long time source records to a DAO. This is the time backward limit for FFTI and cache load data.
*/
public static final int SOURCE_CACHE_TIME = 6;
/**
* The thought was this will eventually be dynamic, static in AWIPS I.
* This is the time back limit for Flash Flood Guidance sources
*/
public static final int FFG_SOURCE_CACHE_TIME = 24;
/** ArrayList of domains to filter for */
private ArrayList<DomainXML> domains = null;
@ -199,7 +217,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
this.pathManager = PathManagerFactory.getPathManager();
statusHandler.handle(Priority.DEBUG, getGeneratorName()
statusHandler.handle(Priority.INFO, getGeneratorName()
+ " process Filter Config...");
domains = new ArrayList<DomainXML>();
boolean configValid = getRunConfig().isPopulated();
@ -418,7 +436,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
} else {
statusHandler
.debug(getGeneratorName()
.info(getGeneratorName()
+ ": templates not loaded yet. Skipping product");
}
}
@ -500,7 +518,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + fftiDone);
} catch (InterruptedException e) {
e.printStackTrace();
statusHandler.handle(Priority.DEBUG,
"Checking status failed!" + e);
}
}
@ -574,16 +593,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements
@Override
public void run() {
try {
logger.debug("ProcessProduct: Starting thread "
+ ffmpProduct.getSourceName());
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Starting thread "
+ ffmpProduct.getSourceName());
process();
logger.debug("ProcessProduct: Finishing thread "
+ ffmpProduct.getSourceName());
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Finishing thread "
+ ffmpProduct.getSourceName());
} catch (Exception e) {
processes.remove(ffmpProduct.getSourceName());
logger.error("ProcessProduct: removed "
+ ffmpProduct.getSourceName());
e.printStackTrace();
statusHandler.handle(Priority.ERROR, "ProcessProduct: removed "
+ ffmpProduct.getSourceName(), e);
}
}
@ -713,7 +735,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
generator, ffmpRec,
ffmp.getFFTISource());
fftiSources.add(ffmp.getFFTISource());
// System.out.println("Adding source to FFTISources!!!!!!!!!!!!"+ffmpRec.getSourceName());
ffti.processFFTI();
}
}
@ -735,7 +756,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
while (productKeys.size() > 0) {
// wait for all threads to finish before returning
try {
Thread.sleep(50);
Thread.sleep(100);
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + productKeys.size());
for (String source : productKeys.keySet()) {
@ -743,7 +764,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
"Still processing ..." + source);
}
} catch (InterruptedException e) {
e.printStackTrace();
statusHandler.handle(Priority.WARN,
"Product Procesing Interrupted! " + e);
}
}
}
@ -865,7 +887,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
try {
Thread.sleep(100);
} catch (InterruptedException e) {
statusHandler.handle(Priority.WARN,
"Domain processing Interrupted!", e);
}
}
}
@ -974,8 +997,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteSourceFileName(sourceId));
try {
sbl = (SourceBinList) SerializationUtil
.transformFromThrift(FileUtil.file2bytes(f.getFile(), true));
sbl = SerializationUtil
.transformFromThrift(SourceBinList.class, FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName());
@ -983,7 +1006,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR,
"Unable to read file " + f.getName());
} catch (IOException ioe) {
ioe.printStackTrace();
statusHandler.handle(Priority.ERROR, "General IO problem with file "
+ f.getName(), ioe);
}
return sbl;
@ -1092,7 +1116,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(
Priority.ERROR,
"Unable to locate new FFG file. "
+ pattern.toString());
+ pattern.toString(), e);
}
}
}
@ -1124,11 +1148,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
siteKey = parts[0];
}
container = loadFFMPDataContainer(siteSourceKey,
hucs, siteKey,
config.getCWA(), backDate);
container = loadFFMPDataContainer(siteSourceKey, hucs, siteKey,
config.getCWA(), backDate);
if (container != null) {
ffmpData.put(siteSourceKey, container);
@ -1200,7 +1221,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR, getGeneratorName()
+ ": filter: " + filter.getName()
+ ": failed to route filter to generator", e);
e.printStackTrace();
}
filter.setValidTime(new Date(System.currentTimeMillis()));
@ -1262,7 +1282,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
boolean write = true;
try {
// write out the fast loader buddy file
// write out the fast loader cache file
long ptime = System.currentTimeMillis();
SourceXML source = getSourceConfig().getSource(
@ -1274,15 +1294,16 @@ public class FFMPGenerator extends CompositeProductGenerator implements
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
// FFG is so infrequent go back a day
backDate = new Date(config.getDate().getTime()
- (3600 * 1000 * 24));
backDate = new Date(
config.getDate().getTime()
- (TimeUtil.MILLIS_PER_HOUR * FFG_SOURCE_CACHE_TIME));
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
backDate = new Date(ffmpRec.getDataTime().getRefTime()
.getTime()
- (3600 * 1000 * 6));
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
// deal with setting of needed HUCS
@ -1336,7 +1357,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (newDate != null && oldDate != null) {
if ((ffmpRec.getDataTime().getRefTime().getTime() - newDate
.getTime()) >= (source
.getExpirationMinutes(ffmpRec.getSiteKey()) * 60 * 1000)) {
.getExpirationMinutes(ffmpRec.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)) {
// force a re-query back to the newest time in
// existing source container, this will fill in
// gaps
@ -1348,10 +1369,12 @@ public class FFMPGenerator extends CompositeProductGenerator implements
ffmpRec.getWfo(), source, ffmpRec
.getSiteKey());
} else if (oldDate.after(new Date(backDate
.getTime()
- (source.getExpirationMinutes(ffmpRec
.getSiteKey()) * 60 * 1000)))) {
} else if (oldDate
.after(new Date(
backDate.getTime()
- (source
.getExpirationMinutes(ffmpRec
.getSiteKey()) * TimeUtil.MILLIS_PER_MINUTE)))) {
// force a re-query back to barrierTime for
// existing source container, this happens if
// the
@ -1423,14 +1446,14 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// this is defensive for if errors get thrown
if (backDate == null) {
backDate = new Date((System.currentTimeMillis())
- (3600 * 1000 * 6));
- (TimeUtil.MILLIS_PER_HOUR * SOURCE_CACHE_TIME));
}
fdc.purge(backDate);
if (write) {
// write it out
writeLoaderBuddyFiles(fdc);
writeCacheFiles(fdc);
}
}
}
@ -1451,37 +1474,31 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long time = System.currentTimeMillis();
FFMPDataContainer fdc = null;
FFMPAggregateRecord record = null;
boolean populated = false;
synchronized (hucs) {
for (String huc : hucs) {
FFMPBasinData basinData = null;
if (checkBuddyFile(sourceSiteDataKey, huc, wfo, backDate)) {
try {
basinData = readLoaderBuddyFile(sourceSiteDataKey, huc,
wfo, backDate);
} catch (Exception e) {
statusHandler.handle(
Priority.ERROR,
"General Error Reading buddy file: "
+ e.getMessage());
}
if (fdc == null) {
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs);
}
}
if (basinData != null) {
fdc.setBasinBuddyData(basinData, huc);
}
if (checkCacheFile(sourceSiteDataKey, wfo, backDate)) {
try {
record = readCacheFile(sourceSiteDataKey, wfo, backDate);
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"General Error Reading cache file: " + e.getMessage());
}
if (fdc == null && record != null) {
// creates a place holder for this source
fdc = new FFMPDataContainer(sourceSiteDataKey, hucs, record);
populated = true;
}
}
if (record != null && !populated) {
fdc.setCacheData(record);
}
if (fdc != null) {
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG, "Loaded Source files: in "
statusHandler.handle(Priority.INFO, "Loaded Source files: in "
+ (time2 - time) + " ms: source: " + sourceSiteDataKey);
}
@ -1489,60 +1506,59 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
/**
* Load existing buddy file
* Load existing cache file
*
* @param sourceSiteDataKey
* @param huc
* @param wfo
* @return
* @throws IOException
*/
private FFMPBasinData readLoaderBuddyFile(String sourceSiteDataKey,
String huc, String wfo, Date backDate) throws IOException {
private FFMPAggregateRecord readCacheFile(String sourceSiteDataKey, String wfo,
Date backDate) throws IOException {
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + "-"
+ huc + ".bin");
FFMPBasinData basinData = null;
BufferedInputStream is = null;
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + ".bin");
FFMPAggregateRecord record = null;
GZIPInputStream gis = null;
try {
is = new BufferedInputStream(new FileInputStream(file));
gis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(file)));
DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift);
basinData = (FFMPBasinData) dsm.deserialize(is);
record = (FFMPAggregateRecord) dsm.deserialize(gis);
} catch (SerializationException e) {
statusHandler
.handle(Priority.ERROR,
"Serialization Error Reading buddy file: "
"Serialization Error Reading cache file: "
+ e.getMessage());
} catch (IOException e) {
statusHandler.handle(Priority.ERROR,
"IO Error Reading buddy file: " + e.getMessage());
"IO Error Reading cache file: " + e.getMessage());
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"General Error Reading buddy file: " + e.getMessage());
"General Error Reading cache file: " + e.getMessage());
} catch (Throwable t) {
statusHandler.handle(Priority.ERROR,
"Bogus Thrift Error Reading buddy file: " + t.getMessage());
"Bogus Thrift Error Reading cache file: " + t.getMessage());
} finally {
if (is != null) {
is.close();
if (gis != null) {
gis.close();
}
}
return basinData;
return record;
}
/**
* Write buddy file
* Write cache file
*
* @param sourceSiteDataKey
* @param huc
* @param wfo
* @return
*/
public void writeLoaderBuddyFiles(FFMPDataContainer fdc) {
public void writeCacheFiles(FFMPDataContainer fdc) {
// Write all huc levels in separate files
File fileDir = new File("" + sharePath + config.getCWA());
@ -1555,7 +1571,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
/**
* Inner class to thread writing of BuddyFiles
* Inner class to thread writing of cache files
*
* @author dhladky
*
@ -1569,7 +1585,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
long time = System.currentTimeMillis();
write();
long time2 = System.currentTimeMillis();
statusHandler.handle(Priority.DEBUG, "Wrote loader files: in "
statusHandler.handle(Priority.INFO, "Wrote cache file: in "
+ (time2 - time) + " ms :" + fdc.getFilePath());
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
@ -1604,61 +1620,69 @@ public class FFMPGenerator extends CompositeProductGenerator implements
if (lockfile.canWrite()) {
// write the lock if we can even write to anything
synchronized (fdc.getKeys()) {
for (String huc : fdc.getKeys()) {
FFMPAggregateRecord cacheRecord = null;
FFMPBasinData fbd = fdc.getBasinData(huc);
synchronized (fdc) {
if (fbd.getBasins().size() > 0) {
cacheRecord = new FFMPAggregateRecord();
// times for Guidance basins will be null
cacheRecord.setTimes(fdc.getOrderedTimes());
String tmpFilePath = fileName + "-" + huc
+ ".tmp";
BufferedOutputStream os = null;
for (Entry<String, FFMPBasinData> entry : fdc
.getBasinMap().entrySet()) {
FFMPBasinData fbd = entry.getValue();
fbd.setCache();
cacheRecord.setBasinData(fbd);
}
}
try {
File file = new File(tmpFilePath);
file.createNewFile();
if (cacheRecord.getBasinsMap().size() > 0) {
if (file.canWrite()) {
os = new BufferedOutputStream(
new FileOutputStream(file));
DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift);
dsm.serialize(fbd, os);
fileNames.put(tmpFilePath, fileName
+ "-" + huc + ".bin");
} else {
statusHandler
.handle(Priority.WARN,
"Can not write buddy file: "
+ file.getAbsolutePath());
}
} catch (SerializationException e) {
statusHandler.handle(Priority.ERROR,
"Serialization Error Writing buddy file: "
+ e.getMessage());
} catch (IOException e) {
statusHandler.handle(Priority.ERROR,
"IO Error Writing buddy file: "
+ e.getMessage());
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"General Error Writing buddy file: "
+ e.getMessage());
} finally {
if (os != null) {
os.close();
}
}
String tmpFilePath = fileName + ".tmp";
GZIPOutputStream gos = null;
try {
File file = new File(tmpFilePath);
file.createNewFile();
if (file.canWrite()) {
gos = new GZIPOutputStream(new BufferedOutputStream(
new FileOutputStream(file)));
DynamicSerializationManager dsm = DynamicSerializationManager
.getManager(SerializationType.Thrift);
dsm.serialize(cacheRecord, gos);
fileNames.put(tmpFilePath, fileName
+ ".bin");
} else {
statusHandler.handle(
Priority.WARN,
"Can not write cache file: "
+ file.getAbsolutePath());
}
} catch (SerializationException e) {
statusHandler.handle(Priority.ERROR,
"Serialization Error Writing cache file: "
+ e.getMessage());
} catch (IOException e) {
statusHandler.handle(
Priority.ERROR,
"IO Error Writing cache file: "
+ e.getMessage());
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"General Error Writing cache file: "
+ e.getMessage());
} finally {
if (gos != null) {
gos.close();
}
}
}
}
} catch (Exception e) {
statusHandler
.handle(Priority.ERROR,
"Error writing Buddy File group: "
+ e.getMessage());
statusHandler.handle(Priority.ERROR,
"Error writing cache File: " + e.getMessage());
} finally {
// rename the files to real path
try {
@ -1684,31 +1708,29 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} catch (Exception e) {
statusHandler.handle(
Priority.ERROR,
"IO Error Renaming buddy file: "
"IO Error Renaming cache file: "
+ e.getMessage());
}
}
} catch (Exception e) {
statusHandler.handle(Priority.ERROR,
"IO Error writing buddy files: " + e.getMessage());
"IO Error writing cache files: " + e.getMessage());
}
}
}
/**
* Load existing buddy file
* Load existing cache file
*
* @param sourceSiteDataKey
* @param huc
* @param wfo
* @return
*/
public boolean checkBuddyFile(String sourceSiteDataKey, String huc,
String wfo, Date backDate) {
public boolean checkCacheFile(String sourceSiteDataKey, String wfo,
Date backDate) {
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + "-"
+ huc + ".bin");
File file = new File(sharePath + wfo + "/" + sourceSiteDataKey + ".bin");
String sourceName = sourceSiteDataKey.split("-")[0];
SourceXML source = getSourceConfig().getSourceByDisplayName(sourceName);
@ -1875,8 +1897,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
getAbsoluteFFTIFileName(fftiName));
try {
ffti = (FFTIData) SerializationUtil.transformFromThrift(FileUtil
.file2bytes(f.getFile(), true));
ffti = SerializationUtil.transformFromThrift(FFTIData.class,
FileUtil.file2bytes(f.getFile(), true));
} catch (FileNotFoundException fnfe) {
statusHandler.handle(Priority.ERROR,
"Unable to locate file " + f.getName(), fnfe);

6
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPConfig.java
View file

@ -221,12 +221,12 @@ public class FFMPConfig {
}
/**
* Get the DHR record
* Get the RADAR record
*
* @param uri
* @return
*/
private Object getDHRRecord(String uri) {
private Object getRADARRecord(String uri) {
Object record = null;
try {
record = ScanCommonUtils.getRadarRecord(uri);
@ -398,7 +398,7 @@ public class FFMPConfig {
} else if (source.getDataType().equals(
FFMPSourceConfigurationManager.DATA_TYPE.RADAR
.getDataType())) {
Object dataObject = getDHRRecord(dataUri);
Object dataObject = getRADARRecord(dataUri);
if (dataObject != null) {
// process as a VGB too

12
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFMPInterpolatedGuidanceDelay.java
View file

@ -22,6 +22,7 @@ package com.raytheon.uf.edex.plugin.ffmp.common;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map.Entry;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPBasin;
@ -30,6 +31,7 @@ import com.raytheon.uf.common.dataplugin.ffmp.FFMPDataContainer;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPGuidanceBasin;
import com.raytheon.uf.common.dataplugin.ffmp.FFMPRecord;
import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
/**
@ -41,6 +43,7 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* ------------ ---------- ----------- --------------------------
* 29 July, 2011 6772 dhladky Initial creation
* 29 July, 2012 578 dhladky memory work
* 27 Jan, 2013 1478 dhladky Changed arraylist to list for times, more constants
* </pre>
*
* @author dhladky
@ -96,15 +99,16 @@ public class FFMPInterpolatedGuidanceDelay {
qpeContainer = generator.getFFMPDataContainer(qpeSource.getSourceName()
+ "-" + siteKey + "-" + siteKey, hucs, backDate);
long expirationTime = qpeSource.getExpirationMinutes(siteKey) * 60 * 1000;
// determine lag_time
long expirationTime = qpeSource.getExpirationMinutes(siteKey)
* TimeUtil.MILLIS_PER_MINUTE;
// determine lag_time
long lagTime = (currentRecord.getDataTime().getRefTime().getTime())
+ (long) (ffgSource.getDurationHour() * 60 * 1000);
+ (long) (ffgSource.getDurationHour() * TimeUtil.MILLIS_PER_MINUTE);
// Determine hour fraction.
int fraction_Hr = (int) (((float) (currentRecord.getDataTime()
.getRefTime().getTime() - (lagTime - guidFrequency))) / (float) guidFrequency);
// Gets the ordered times for QPE
ArrayList<Date> orderedTimes = qpeContainer
List<Date> orderedTimes = qpeContainer
.getOrderedTimes(currentRecord.getDataTime().getRefTime());
// EQUATION: Guid = GuidOld + R i/d (GuidNew - GuidOld)

33
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/common/FFTI.java
View file

@ -44,6 +44,7 @@ import com.raytheon.uf.common.monitor.xml.SourceXML;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
/**
@ -54,8 +55,8 @@ import com.raytheon.uf.edex.plugin.ffmp.FFMPGenerator;
* SOFTWARE HISTORY
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 01, 2011 dhladky Initial creation
* July 13, 2012 dhladky Revamped to help memory
* Apr 01, 2011 dhladky Initial creation
* July 13, 2012 dhladky Revamped to help memory
*
* </pre>
*
@ -443,7 +444,6 @@ public class FFTI implements Runnable {
"Failed to evaluate Ratio/Diff. "
+ attribute.getAttributeName() + ": " + displayName
+ "\n" + e);
e.printStackTrace();
}
}
@ -826,9 +826,9 @@ public class FFTI implements Runnable {
}
long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000);
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * 60 * 1000;
long expirationTime = ffmpSource.getExpirationMinutes(fftiSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
FFMPDataContainer fdc = null;
@ -894,6 +894,15 @@ public class FFTI implements Runnable {
return accumulator;
}
/**
* Gets the ratio and difference values for this site
* @param qSourceKey
* @param qSiteKey
* @param ffgType
* @param duration
* @param unit
* @return
*/
private FFTIRatioDiff getRatioAndDiffForSite(String qSourceKey,
String qSiteKey, String ffgType, double duration, String unit) {
@ -923,13 +932,13 @@ public class FFTI implements Runnable {
values.setUnit(unit);
long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000);
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * 60 * 1000;
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
// make sure we have data
Date ffgBackDate = new Date(config.getDate().getTime()
- (3600 * 1000 * 24));
- (TimeUtil.MILLIS_PER_HOUR * FFMPGenerator.FFG_SOURCE_CACHE_TIME));
String primarySource = ffmpgen.fscm.getPrimarySource(ffmpQSource);
ProductXML product = ffmpgen.fscm.getProduct(primarySource);
@ -950,7 +959,7 @@ public class FFTI implements Runnable {
if (guidSourceExpiration == 0l) {
guidSourceExpiration = iguidSource
.getExpirationMinutes(qSiteKey) * 60 * 1000;
.getExpirationMinutes(qSiteKey) * TimeUtil.MILLIS_PER_MINUTE;
break;
}
}
@ -1046,7 +1055,7 @@ public class FFTI implements Runnable {
SourceXML ffmpQSource, double duration, String qSiteKey) {
long cur = config.getDate().getTime();
long timeBack = (long) (duration * 3600 * 1000);
long timeBack = (long) (duration * TimeUtil.MILLIS_PER_HOUR);
Date backDate = new Date(cur - timeBack);
long expirationTime = ffmpQSource.getExpirationMinutes(qSiteKey);
Double gapVal = 0.0;
@ -1054,14 +1063,14 @@ public class FFTI implements Runnable {
if (qpeContainer.getOrderedTimes(backDate) != null) {
gapVal = 0.0;
ArrayList<FFMPGap> gaps = FFMPGap.getGaps(
List<FFMPGap> gaps = FFMPGap.getGaps(
qpeContainer.getOrderedTimes(backDate), expirationTime,
backDate, config.getDate());
for (FFMPGap gap : gaps) {
gapVal += gap.getGap();
}
gapVal = gapVal / 60;
gapVal = gapVal / TimeUtil.MINUTES_PER_HOUR;
}
return gapVal;

7
hebbard_deletes.txt
View file

@ -1,7 +0,0 @@
cave/build/static/common/cave/etc/ncep/PgenXmlOverlayProducts
cave/build/static/common/cave/etc/ncep/ResourceDefns/OVERLAYS/G2tAtl
cave/build/static/common/cave/etc/ncep/ResourceDefns/OVERLAYS/G2tPac
cave/build/static/common/cave/etc/ncep/ResourceDefns/OVERLAYS/G2tTpc
ncep/gov.noaa.nws.ncep.edex.plugin.convsigmet/gov.noaa.nws.ncep.edex.plugin.convsigmet
ncep/gov.noaa.nws.ncep.edex.plugin.nonconvsigmet/gov.noaa.nws.ncep.edex.plugin.nonconvsigmet
ncep/gov.noaa.nws.ncep.viz.rsc.ncgrid/src/gov/noaa/nws/ncep/viz/rsc/ncgrid/util

74
ncep/gov.noaa.nws.ncep.ui.nsharp/src/gov/noaa/nws/ncep/ui/nsharp/display/rsc/NsharpResourceHandler.java
View file

@ -1564,11 +1564,11 @@ public class NsharpResourceHandler {
addElementToTableAndLists(elmDesc,stnId,timeLine,stnInfo);
}
if(displayNewData){
//Set default parcel trace data
//Set default parcel trace data
currentParcel = NsharpNativeConstants.PARCELTYPE_MOST_UNSTABLE;
currentParcelLayerPressure = NsharpNativeConstants.MU_LAYER;
setCurrentSoundingLayerInfo();
resetData();
setCurrentSoundingLayerInfo();
resetData();
}
else {
//Not display new data. Reset current "parameter"s after adding data to map/lists
@ -1580,11 +1580,21 @@ public class NsharpResourceHandler {
//set total time line group and stn id list page number
int numTimeLinePerPage = (cnYOrig-dtNextPageEnd)/charHeight;
//System.out.println("numTimeLinePerPage="+numTimeLinePerPage);
totalTimeLinePage = timeLineStateList.size()/numTimeLinePerPage + 1; //NEW CODE
curTimeLinePage = currentTimeLineStateListIndex/numTimeLinePerPage + 1; //NEW CODE
totalStnIdPage = stnStateList.size()/numTimeLinePerPage + 1; //NEW CODE
curStnIdPage= currentStnStateListIndex/numTimeLinePerPage + 1; //NEW CODE
//fix bug, when numTimeLinePerPage ==0 case
if(numTimeLinePerPage <= 0) {
numTimeLinePerPage = 1;
totalTimeLinePage = timeLineStateList.size();
curTimeLinePage = currentTimeLineStateListIndex;
totalStnIdPage = stnStateList.size();
curStnIdPage= currentStnStateListIndex;
}
else{
totalTimeLinePage = timeLineStateList.size()/numTimeLinePerPage + 1;
curTimeLinePage = currentTimeLineStateListIndex/numTimeLinePerPage + 1;
totalStnIdPage = stnStateList.size()/numTimeLinePerPage + 1;
curStnIdPage= currentStnStateListIndex/numTimeLinePerPage + 1;
}
/* Chin: TBD: do we need these code?
NsharpSkewTPaneDisplay renderableDisplay = (NsharpSkewTPaneDisplay) skewtPaneRsc.getDescriptor().getRenderableDisplay();
@ -1650,9 +1660,9 @@ public class NsharpResourceHandler {
if( totalStnIdPage == 1)
return;
if((c.x - (dtXOrig+dtWidth)) < (dtWidth/2)){
curStnIdPage++;
if(curStnIdPage>totalStnIdPage)
curStnIdPage=1;
curStnIdPage++;
if(curStnIdPage>totalStnIdPage)
curStnIdPage=1;
} else {
curStnIdPage--;
if(curStnIdPage <=0)
@ -1695,9 +1705,9 @@ public class NsharpResourceHandler {
if( totalTimeLinePage == 1)
return;
if((c.x - dtXOrig) < (dtWidth/2)){
curTimeLinePage++;
if(curTimeLinePage>totalTimeLinePage)
curTimeLinePage=1;
curTimeLinePage++;
if(curTimeLinePage>totalTimeLinePage)
curTimeLinePage=1;
} else {
curTimeLinePage--;
if(curTimeLinePage <=0)
@ -1709,7 +1719,7 @@ public class NsharpResourceHandler {
index =((int)(c.y - dtNextPageEnd))/ charHeight +
(curTimeLinePage-1)* numTimeLinePerPage ;
if( index < timeLineStateList.size() ){
if( index < timeLineStateList.size() && index >=0 ){
switch(timeLineStateList.get(index).getTimeState()){
case INACTIVE:
timeLineStateList.get(index).setTimeState( NsharpConstants.State.ACTIVE);
@ -2177,28 +2187,28 @@ public class NsharpResourceHandler {
NsharpAbstractPaneResource absPaneRsc = (NsharpAbstractPaneResource)rscP.getResource();
if (absPaneRsc instanceof NsharpSkewTPaneResource){
skewtPaneRsc = (NsharpSkewTPaneResource)absPaneRsc ;
skewtPaneRsc.setLinePropertyMap(linePropertyMap);
skewtPaneRsc.setGraphConfigProperty(graphConfigProperty);
skewtPaneRsc.setNsharpNative(nsharpNative);
}
skewtPaneRsc.setLinePropertyMap(linePropertyMap);
skewtPaneRsc.setGraphConfigProperty(graphConfigProperty);
skewtPaneRsc.setNsharpNative(nsharpNative);
}
else if (absPaneRsc instanceof NsharpDataPaneResource){
dataPaneRsc = (NsharpDataPaneResource)absPaneRsc;
dataPaneRsc.setLinePropertyMap(linePropertyMap);
dataPaneRsc.setGraphConfigProperty(graphConfigProperty);
dataPaneRsc.setNsharpNative(nsharpNative);
dataPaneRsc.setPageDisplayOrderNumberArray(pageDisplayOrderNumberArray);
}
dataPaneRsc.setLinePropertyMap(linePropertyMap);
dataPaneRsc.setGraphConfigProperty(graphConfigProperty);
dataPaneRsc.setNsharpNative(nsharpNative);
dataPaneRsc.setPageDisplayOrderNumberArray(pageDisplayOrderNumberArray);
}
else if (absPaneRsc instanceof NsharpHodoPaneResource){
hodoPaneRsc = (NsharpHodoPaneResource)absPaneRsc;
hodoPaneRsc.setLinePropertyMap(linePropertyMap);
hodoPaneRsc.setGraphConfigProperty(graphConfigProperty);
hodoPaneRsc.setNsharpNative(nsharpNative);
}
hodoPaneRsc.setLinePropertyMap(linePropertyMap);
hodoPaneRsc.setGraphConfigProperty(graphConfigProperty);
hodoPaneRsc.setNsharpNative(nsharpNative);
}
else if (absPaneRsc instanceof NsharpWitoPaneResource &&
(paneConfigurationName.equals(NsharpConstants.PANE_SPCWS_CFG_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_1_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_1_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_2_STR))){
witoPaneRsc = (NsharpWitoPaneResource)absPaneRsc;
witoPaneRsc.setLinePropertyMap(linePropertyMap);
witoPaneRsc.setGraphConfigProperty(graphConfigProperty);
@ -2214,7 +2224,7 @@ public class NsharpResourceHandler {
insetPaneRsc.setLinePropertyMap(linePropertyMap);
insetPaneRsc.setGraphConfigProperty(graphConfigProperty);
insetPaneRsc.setNsharpNative(nsharpNative);
}
else if (absPaneRsc instanceof NsharpSpcGraphsPaneResource && paneConfigurationName.equals(NsharpConstants.PANE_SPCWS_CFG_STR)){
spcGraphsPaneRsc = (NsharpSpcGraphsPaneResource)absPaneRsc;
@ -2231,7 +2241,7 @@ public class NsharpResourceHandler {
}
else if (absPaneRsc instanceof NsharpTimeStnPaneResource &&
(paneConfigurationName.equals(NsharpConstants.PANE_SIMPLE_D2D_CFG_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_1_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_1_STR)||
paneConfigurationName.equals(NsharpConstants.PANE_DEF_CFG_2_STR))){
timeStnPaneRsc = (NsharpTimeStnPaneResource)absPaneRsc ;
timeStnPaneRsc.setLinePropertyMap(linePropertyMap);