xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

13.2.1-15 baseline

Browse source 
Former-commit-id: b16f2448a9 [formerly ff687e572b] [formerly 7b52d3eb6e] [formerly 7617a2437f [formerly 7b52d3eb6e [formerly a6d7a7ca85cc185d9cd3394a6e6e82f72b8f70fc]]]
Former-commit-id: 7617a2437f
Former-commit-id: 95795084a4154ad9fabef9bc6d518b7567cc0437 [formerly 29c9d5a170]
Former-commit-id: b237f19e78
This commit is contained in:
Steve Harris 2013-03-18 12:53:04 -04:00
parent 874db8cbea
commit 75e51266e4
4 changed files with 77 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
edexOsgi/com.raytheon.uf.edex.grid.staticdata/src/com/raytheon/uf/edex/grid/staticdata/StaticDataGenerator.java
View file

@ -70,6 +70,7 @@ import com.raytheon.uf.edex.plugin.grid.dao.GridDao;
* ------------ ---------- ----------- --------------------------
* Dec 3, 2010 rjpeter Initial creation
* Mar 07, 2013 1587 bsteffen rewrite static data generation.
* Mar 14, 2013 1587 bsteffen Fix persisting to datastore.
*
* </pre>
*
@ -267,7 +268,7 @@ public class StaticDataGenerator {
for (GridRecord staticRecord : datastoreRecords) {
populateMessageData(staticRecord);
}
dao.persistToHDF5(databaseRecords.toArray(new PluginDataObject[0]));
dao.persistToHDF5(datastoreRecords.toArray(new PluginDataObject[0]));
}
if (!databaseRecords.isEmpty()) {
dao.persistToDatabase(databaseRecords
@ -423,7 +424,8 @@ public class StaticDataGenerator {
datasets = Collections.emptyList();
}
}
if (datasets.contains(missing)) {
if (!datasets.contains(staticRecord.getParameter()
.getAbbreviation())) {
missing.add(staticRecord);
}
}

4
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/res/spring/ffmp-ingest.xml
View file

@ -8,8 +8,8 @@
<bean id="ffmpThreadPool"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="2" />
<property name="maxPoolSize" value="4" />
<property name="corePoolSize" value="1" />
<property name="maxPoolSize" value="1" />
<property name="keepAliveSeconds" value="60000" />
</bean>

75
edexOsgi/com.raytheon.uf.edex.plugin.ffmp/src/com/raytheon/uf/edex/plugin/ffmp/FFMPGenerator.java
View file

@ -118,6 +118,7 @@ import com.raytheon.uf.edex.plugin.ffmp.common.FFTIRatioDiff;
* 07/31/2011 578 dhladky FFTI modifications
* 01/27/13 1478 D. Hladky Added creation of full cache records to help read write stress on NAS
* 02/25/13 1660 D. Hladky Redesigned data flow for FFTI in order to have only one mosaic piece in memory at a time.
* 03/13/13 1478 D. Hladky non-FFTI mosaic containers weren't getting ejected. Made it so that they are ejected after processing as well.
* </pre>
*
* @author dhladky
@ -701,18 +702,47 @@ public class FFMPGenerator extends CompositeProductGenerator implements
fftiSources.add(ffmp.getFFTISource());
ffti.processFFTI();
}
// Do the accumulation now, more memory efficient.
// Do the accumulation now, more memory efficient.
// Only one piece in memory at a time
for (String attribute: ffmp.getAttributes()) {
if (attribute.equals(ATTRIBUTE.ACCUM.getAttribute())) {
FFTIAccum accum = getAccumulationForSite(ffmpProduct.getDisplayName(), siteKey, dataKey, fftiSource.getDurationHour(), ffmpProduct.getUnit(siteKey));
if (statusHandler.isPriorityEnabled(Priority.DEBUG)) {
statusHandler.debug("Accumulating FFTI for source: "+ffmpProduct.getDisplayName()+" site: "+siteKey+" data: "+dataKey+" duration: "+fftiSource.getDurationHour()+ " accumulation: "+accum.getAccumulation());
for (String attribute : ffmp.getAttributes()) {
if (attribute.equals(ATTRIBUTE.ACCUM
.getAttribute())) {
FFTIAccum accum = getAccumulationForSite(
ffmpProduct.getDisplayName(),
siteKey, dataKey,
fftiSource.getDurationHour(),
ffmpProduct.getUnit(siteKey));
if (statusHandler
.isPriorityEnabled(Priority.DEBUG)) {
statusHandler
.debug("Accumulating FFTI for source: "
+ ffmpProduct
.getDisplayName()
+ " site: "
+ siteKey
+ " data: "
+ dataKey
+ " duration: "
+ fftiSource
.getDurationHour()
+ " accumulation: "
+ accum.getAccumulation());
}
}
}
}
}
SourceXML source = getSourceConfig().getSource(
ffmpRec.getSourceName());
if (!source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
String sourceSiteDataKey = getSourceSiteDataKey(source,
dataKey, ffmpRec);
ffmpData.remove(sourceSiteDataKey);
statusHandler.info("Removing from memory: "+sourceSiteDataKey);
}
}
}
}
@ -1918,7 +1948,6 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
ffmpData.remove(siteDataKey);
accumulator.setReset(false);
writeFFTIData(siteDataKey, accumulator);
}
@ -2102,5 +2131,33 @@ public class FFMPGenerator extends CompositeProductGenerator implements
}
}
/**
* Find siteSourceDataKey
*
* @param source
* @param dataKey
* @param ffmpRec
* @return
*/
private String getSourceSiteDataKey(SourceXML source, String dataKey, FFMPRecord ffmpRec) {
String sourceName = source.getSourceName();
String sourceSiteDataKey = null;
if (source.getSourceType().equals(
SOURCE_TYPE.GUIDANCE.getSourceType())) {
sourceName = source.getDisplayName();
sourceSiteDataKey = sourceName;
} else {
sourceName = ffmpRec.getSourceName();
sourceSiteDataKey = sourceName + "-" + ffmpRec.getSiteKey()
+ "-" + dataKey;
}
return sourceSiteDataKey;
}
}

6
edexOsgi/com.raytheon.uf.edex.plugin.grid/src/com/raytheon/uf/edex/plugin/grid/dao/GridDao.java
View file

@ -66,6 +66,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* 4/7/09 1994 bphillip Initial Creation
* Mar 14, 2013 1587 bsteffen Fix static data persisting to datastore.
*
* </pre>
*
@ -99,10 +100,13 @@ public class GridDao extends PluginDao {
long[] sizes = new long[] { location.getNx(), location.getNy() };
String abbrev = gridRec.getParameter().getAbbreviation();
String group = gridRec.getDataURI();
String datasetName = "Data";
if (GridPathProvider.STATIC_PARAMETERS.contains(abbrev)) {
group = "/" + location.getId();
datasetName = abbrev;
}
AbstractStorageRecord storageRecord = new FloatDataRecord("Data",
AbstractStorageRecord storageRecord = new FloatDataRecord(
datasetName,
group, (float[]) messageData, 2, sizes);
storageRecord.setCorrelationObject(gridRec);