xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1635 Second Guidance source verification, general cleanup" into development

Browse source 
Former-commit-id: 73f53bd513 [formerly a9c4e54287 [formerly 25ecfc3e6199294275abc8547c482388c55be724]]
Former-commit-id: a9c4e54287
Former-commit-id: 033b9e3852
This commit is contained in:
Nate Jensen 2013-02-21 11:41:52 -06:00 • committed by Gerrit Code Review
commit a8e81097fc
8 changed files with 1286 additions and 1307 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/FFMPMonitor.java
View file

@ -91,6 +91,7 @@ import com.raytheon.uf.viz.monitor.listeners.IMonitorListener;
* 02/01/13 1569 D. Hladky updated constants
* 02/01/13 1627 D. Hladky removed unused(useless) db load method
* 02/19/13 1639 njensen Replaced ConcurrentHashMaps with data structures
* 02/20/13 1635 D. Hladky Fixed multi guidance sources
*
* </pre>
*
@ -119,23 +120,6 @@ public class FFMPMonitor extends ResourceMonitor {
private FFMPSiteDataContainer siteDataMap = new FFMPSiteDataContainer();
// /** FFMP Records indexed by site, times and field **/
// public ConcurrentHashMap<String, ConcurrentHashMap<String,
// FFMPCacheRecord>> ffmpData = null;
//
// // list of earliest available date queried by site, sourceName
// public ConcurrentHashMap<String, ConcurrentHashMap<String, Date>>
// ffmpAvailableUriQueryDates = null;
//
// // map by site, sourceName, Date
// public ConcurrentHashMap<String, ConcurrentHashMap<String,
// ConcurrentSkipListMap<Date, List<String>>>> ffmpAvailableUris = null;
//
// // map by field, huc, and the URIs that have been loaded for
// public ConcurrentHashMap<String, ConcurrentHashMap<String,
// ConcurrentHashMap<String, ConcurrentHashMap<String, String>>>>
// ffmpLoadedUris = null;
// Interpolation Guidance Sources
public FFMPGuidanceInterpolation interpolation = null;
@ -890,7 +874,7 @@ public class FFMPMonitor extends ResourceMonitor {
ffmpSplash.close();
break;
}
Thread.sleep(1000);
Thread.sleep(100);
} catch (InterruptedException e) {
e.printStackTrace();
if (ffmpSplash != null) {

26
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/dialogs/FfmpBasinTableDlg.java
View file

@ -1122,19 +1122,21 @@ public class FfmpBasinTableDlg extends CaveSWTDialog implements
private void updateGapValueLabel(double gapVal) {
if (gapVal > 0.75) {
gapValueLbl.setBackground(getDisplay()
.getSystemColor(SWT.COLOR_RED));
} else if (gapVal > 0.30) {
gapValueLbl.setBackground(getDisplay().getSystemColor(
SWT.COLOR_YELLOW));
} else {
gapValueLbl.setBackground(getDisplay().getSystemColor(
SWT.COLOR_GREEN));
}
if (gapValueLbl != null) {
if (gapVal > 0.75) {
gapValueLbl.setBackground(getDisplay().getSystemColor(
SWT.COLOR_RED));
} else if (gapVal > 0.30) {
gapValueLbl.setBackground(getDisplay().getSystemColor(
SWT.COLOR_YELLOW));
} else {
gapValueLbl.setBackground(getDisplay().getSystemColor(
SWT.COLOR_GREEN));
}
String str = String.format("%2.2f ", gapVal);
gapValueLbl.setText(str);
String str = String.format("%2.2f ", gapVal);
gapValueLbl.setText(str);
}
}
private void initTimeDuration() {

4
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/dialogs/FfmpTableConfigData.java
View file

@ -20,6 +20,7 @@
package com.raytheon.uf.viz.monitor.ffmp.ui.dialogs;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@ -46,8 +47,10 @@ import com.raytheon.uf.viz.monitor.ffmp.xml.FFMPTableColumnXML;
* ------------ ---------- ----------- --------------------------
* Mar 8, 2011 lvenable Initial creation
* Mar 8, 2012 DR 14406 gzhang Fixing QPF Column Title Missing
* Feb 19, 2013 1635 dhladky Fixed multiple guidance displays
* </pre>
*
*
* @author lvenable
* @version 1.0
*/
@ -261,6 +264,7 @@ public class FfmpTableConfigData {
}
ArrayList<String> guidTypes = productXml.getAvailableGuidanceTypes();
Collections.sort(guidTypes);
ffmpTableColMap.put(COLUMN_NAME.NAME.getColumnName(),
COLUMN_NAME.NAME.getColumnName());

2391
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPDataGenerator.java
View file

File diff suppressed because it is too large Load diff

62
cave/com.raytheon.uf.viz.monitor.ffmp/src/com/raytheon/uf/viz/monitor/ffmp/ui/rsc/FFMPResource.java
View file

@ -160,6 +160,7 @@ import com.vividsolutions.jts.geom.Point;
* 02/01/13 1569 D. Hladky Added constants
* 10 Feb 2013 1584 mpduff Add performance logging.
* Feb 19, 2013 1639 njensen Replaced FFMPCacheRecord with FFMPRecord
* Feb 20, 2013 1635 dhladky Fixed multiple guidance display
* </pre>
*
* @author dhladky
@ -475,7 +476,7 @@ public class FFMPResource extends
} else {
while (!loader.isDone) {
try {
Thread.sleep(1000);
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
}
@ -487,7 +488,7 @@ public class FFMPResource extends
while (!loader.isDone) {
try {
Thread.sleep(1000);
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
}
@ -708,7 +709,7 @@ public class FFMPResource extends
value = getGuidanceRecord().getBasinData(FFMPRecord.ALL)
.getMaxGuidanceValue(pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration(), fips);
getGuidSourceExpiration(getFFGName()), fips);
break;
}
case QPE: {
@ -803,6 +804,7 @@ public class FFMPResource extends
private float forceValue(List<Long> pfafs, FFMPBasin basin,
float unforcedValue) {
float value = unforcedValue;
String ffgType = getFFGName();
if (forceUtil == null) {
forceUtil = new FFFGForceUtil(this, getFFGName());
@ -821,15 +823,15 @@ public class FFMPResource extends
// Recalculate the guidance using the forced value(s)
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(getFFGName()),
this.getGuidanceInterpolation(ffgType),
new Float(value), forcedPfafs,
getGuidSourceExpiration());
getGuidSourceExpiration(ffgType));
} else if (forcedPfafs.size() > 0) {
value = guidRecord.getBasinData(FFMPRecord.ALL)
.getAverageGuidanceValue(pfafList,
this.getGuidanceInterpolation(getFFGName()),
this.getGuidanceInterpolation(ffgType),
Float.NaN, forcedPfafs,
getGuidSourceExpiration());
getGuidSourceExpiration(ffgType));
}
}
@ -1870,6 +1872,7 @@ public class FFMPResource extends
float qpe = 0.0f;
float guid = 0.0f;
float diff = Float.NaN;
String ffgType = getFFGName();
try {
if (aggregate) {
@ -1886,8 +1889,8 @@ public class FFMPResource extends
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getGuidanceValues(pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration());
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
}
if ((qpes != null) && (guids != null)) {
diff = FFMPUtils.getMaxDiffValue(qpes, guids);
@ -1908,7 +1911,7 @@ public class FFMPResource extends
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, getFFGName());
recentTime, ffgType);
diff = FFMPUtils.getDiffValue(qpe, guid);
}
@ -1924,7 +1927,7 @@ public class FFMPResource extends
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
recentTime, getFFGName());
recentTime, ffgType);
guid = forceValue(pfafs,
getBasin(key, getField(), recentTime, aggregate),
guid);
@ -1932,7 +1935,7 @@ public class FFMPResource extends
}
}
} catch (Exception e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR, "Error caculating Diff", e);
}
return diff;
}
@ -1950,6 +1953,7 @@ public class FFMPResource extends
float qpe = 0.0f;
float guid = 0.0f;
float ratio = Float.NaN;
String ffgType = getFFGName();
try {
if (aggregate) {
@ -1966,8 +1970,8 @@ public class FFMPResource extends
guids = getGuidanceRecord()
.getBasinData(FFMPRecord.ALL)
.getGuidanceValues(pfafs,
getGuidanceInterpolation(getFFGName()),
getGuidSourceExpiration());
getGuidanceInterpolation(ffgType),
getGuidSourceExpiration(ffgType));
}
if ((qpes != null) && (guids != null)) {
ratio = FFMPUtils.getMaxRatioValue(qpes, guids);
@ -1987,7 +1991,7 @@ public class FFMPResource extends
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(getHuc()).get(key),
recentTime, getFFGName());
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
}
}
@ -2001,12 +2005,12 @@ public class FFMPResource extends
guid = getGuidanceValue(
(FFMPGuidanceBasin) getGuidanceRecord()
.getBasinData(FFMPRecord.ALL).get(key),
recentTime, getFFGName());
recentTime, ffgType);
ratio = FFMPUtils.getRatioValue(qpe, guid);
}
}
} catch (Exception e) {
e.printStackTrace();
statusHandler.handle(Priority.ERROR, "Error caculating Ratio", e);
}
return ratio;
}
@ -3426,14 +3430,14 @@ public class FFMPResource extends
for (SourceXML ffgSource : getProduct().getGuidanceSourcesByType(
ffgGraphType)) {
if (guidBasin.getValue(ffgSource.getSourceName(),
guidanceInterpolator, getGuidSourceExpiration()) != null) {
guidanceInterpolator, getGuidSourceExpiration(ffgGraphType)) != null) {
double time = FFMPGuiUtils.getTimeDiff(mostRecentRefTime,
FFMPGuiUtils.getHourDisplacement(mostRecentRefTime,
ffgSource.getDurationHour()));
fgd.setGuid(time, (double) guidBasin.getValue(
ffgSource.getSourceName(), guidanceInterpolator,
getGuidSourceExpiration()));
getGuidSourceExpiration(ffgGraphType)));
guidTimes.add(time);
}
}
@ -3503,13 +3507,13 @@ public class FFMPResource extends
guidanceInterpolator
.getInterpolationOffset(),
guidanceInterpolator,
getGuidSourceExpiration());
getGuidSourceExpiration(ffgGraphType));
} else {
if (guidanceInterpolator.getSource1() != null) {
guidancev = guidBasin.getValue(
guidanceInterpolator.getSource1(),
guidanceInterpolator,
getGuidSourceExpiration());
getGuidSourceExpiration(ffgGraphType));
}
}
@ -3934,12 +3938,12 @@ public class FFMPResource extends
getGuidanceInterpolation(guidType)
.getInterpolationOffset(),
getGuidanceInterpolation(guidType),
getGuidSourceExpiration());
getGuidSourceExpiration(guidType));
} else {
dvalue = basin.getValue(getGuidanceInterpolation(guidType)
.getStandardSource(),
getGuidanceInterpolation(guidType),
getGuidSourceExpiration());
getGuidSourceExpiration(guidType));
}
if (dvalue == FFMPUtils.MISSING) {
@ -3948,7 +3952,7 @@ public class FFMPResource extends
} else {
dvalue = basin.getValue(getPrimarySource(), recentTime,
getGuidanceInterpolation(guidType),
getGuidSourceExpiration());
getGuidSourceExpiration(guidType));
}
}
@ -4125,18 +4129,12 @@ public class FFMPResource extends
*
* @return
*/
public long getGuidSourceExpiration() {
public long getGuidSourceExpiration(String guidType) {
if (guidSourceExpiration == 0l) {
if (getProduct() != null) {
String guidSrc = FFMPConfig.getInstance().getFFMPConfigData()
.getIncludedGuids();
if (guidSrc.contains(",")) {
String[] parts = guidSrc.split(",");
guidSrc = parts[0];
}
SourceXML source = getProduct().getGuidanceSourcesByType(
guidSrc).get(0);
guidType).get(0);
guidSourceExpiration = source
.getExpirationMinutes(getSiteKey())
* TimeUtil.MILLIS_PER_MINUTE;

8
cave/com.raytheon.uf.viz.stats/src/com/raytheon/uf/viz/stats/collector/StatsCollector.java
View file

@ -35,6 +35,7 @@ import com.raytheon.uf.common.stats.StatisticsEvent;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Feb 8, 2013 mpduff Initial creation
* Feb 19, 2013 1635 dhladky Safety from null
*
* </pre>
*
@ -106,8 +107,9 @@ public class StatsCollector {
*/
public void setStop(String key) {
StatisticsEvent event = statsDataMap.remove(key);
event.finalizeEvent();
EventBus.publish(event);
if (event != null) {
event.finalizeEvent();
EventBus.publish(event);
}
}
}

3
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/FFMPTemplates.java
View file

@ -83,6 +83,7 @@ import com.vividsolutions.jts.io.WKBReader;
* 07/09/10 3914 D. Hladky Localization work
* 12/13/10 7484 D. Hladky Service Backup
* 02/01/13 1569 D.Hladky Constants
* 02/20/13 1635 D. Hladky Constants
*
* </pre>
*
@ -568,7 +569,7 @@ public class FFMPTemplates {
"FFMPTemplate: Starting site template process " + dataKey
+ ":" + cwa + ":" + huc);
// special handling for VGB's
if (huc.equals("VIRTUAL")) {
if (huc.equals(FFMPRecord.VIRTUAL)) {
getVirtualGageBasins(dataKey, cwa);
} else {
getMap(dataKey, cwa, huc);

79
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -111,6 +111,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIProcessor;
* 07/31/2011 578 dhladky FFTI modifications
* 01/27/13 1478 D. Hladky Added creation of full cache records to help read write stress on NAS
* 02/01/13 1569 D. Hladky Added constants, switched to using aggregate records written through pypies
* 02/20/13 1635 D. Hladky Added some finally methods to increase dead lock safety. Reduced wait times for threads.
* </pre>
*
* @author dhladky
@ -468,15 +469,19 @@ public class FFMPGenerator extends CompositeProductGenerator implements
while (processes.size() > 0) {
// wait for all threads to finish before returning
try {
Thread.sleep(100);
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + processes.size());
for (String source : processes.keySet()) {
Thread.sleep(50);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.handle(Priority.DEBUG,
"Still processing ..." + source);
"Checking status ..." + processes.size());
for (String source : processes.keySet()) {
statusHandler.handle(Priority.DEBUG,
"Still processing ..." + source);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
statusHandler.handle(Priority.WARN,
"Thread interrupted ..." + e);
}
}
@ -506,9 +511,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
while (fftiSources.size() > 0) {
try {
Thread.sleep(100);
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + fftiDone);
Thread.sleep(50);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + fftiDone);
}
} catch (InterruptedException e) {
statusHandler.handle(Priority.DEBUG,
"Checking status failed!" + e);
@ -532,8 +539,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
} catch (Throwable e) {
statusHandler.handle(Priority.ERROR,
"Unable to process FFMP Records.");
e.printStackTrace();
"Unable to process FFMP Records.", e);
}
}
}
@ -585,19 +591,25 @@ public class FFMPGenerator extends CompositeProductGenerator implements
@Override
public void run() {
try {
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Starting thread "
+ ffmpProduct.getSourceName());
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Starting thread "
+ ffmpProduct.getSourceName());
}
process();
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Finishing thread "
+ ffmpProduct.getSourceName());
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.handle(
Priority.DEBUG,
"ProcessProduct: Finishing thread "
+ ffmpProduct.getSourceName());
}
} catch (Exception e) {
processes.remove(ffmpProduct.getSourceName());
statusHandler.handle(Priority.ERROR, "ProcessProduct: removed "
+ ffmpProduct.getSourceName(), e);
} finally {
// resets the process list
processes.remove(ffmpProduct.getSourceName());
}
}
@ -748,12 +760,15 @@ public class FFMPGenerator extends CompositeProductGenerator implements
while (productKeys.size() > 0) {
// wait for all threads to finish before returning
try {
Thread.sleep(100);
statusHandler.handle(Priority.DEBUG,
"Checking status ..." + productKeys.size());
for (String source : productKeys.keySet()) {
Thread.sleep(50);
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.handle(Priority.DEBUG,
"Still processing ..." + source);
"Checking status ..." + productKeys.size());
for (String source : productKeys.keySet()) {
statusHandler.handle(Priority.DEBUG,
"Still processing ..." + source);
}
}
} catch (InterruptedException e) {
statusHandler.handle(Priority.WARN,
@ -767,7 +782,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
recs[i] = ffmpRecords.get(i);
}
products.put(ffmpProduct.getSourceName(), recs);
processes.remove(ffmpProduct.getSourceName());
}
}
@ -811,8 +825,8 @@ public class FFMPGenerator extends CompositeProductGenerator implements
*/
public void createUnifiedGeometries(DomainXML domain) {
ArrayList<String> hucsToGen = new ArrayList<String>();
hucsToGen.add("ALL");
hucsToGen.add("COUNTY");
hucsToGen.add(FFMPRecord.ALL);
hucsToGen.add(FFMPRecord.COUNTY);
for (int i = template.getTotalHucLevels() - 1; i >= 0; i--) {
hucsToGen.add("HUC" + i);
@ -877,7 +891,7 @@ public class FFMPGenerator extends CompositeProductGenerator implements
// Didn't process a domain, locked by another cluster
// member, sleep and try again
try {
Thread.sleep(100);
Thread.sleep(50);
} catch (InterruptedException e) {
statusHandler.handle(Priority.WARN,
"Domain processing Interrupted!", e);
@ -1204,10 +1218,11 @@ public class FFMPGenerator extends CompositeProductGenerator implements
statusHandler.handle(Priority.ERROR, getGeneratorName()
+ ": filter: " + filter.getName()
+ ": failed to route filter to generator", e);
} finally {
// safer, this way filter never gets set in weird state
filter.setValidTime(new Date(System.currentTimeMillis()));
filter.reset();
}
filter.setValidTime(new Date(System.currentTimeMillis()));
filter.reset();
}
/**