xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #1937: Speed up ifpnetCDF by using WECache to prefetch grid/histories as in A1.

Browse source 
Change-Id: I842562407cf93f38ea7b64d6fc739503f02c4c39

Former-commit-id: e203ff703fa1d0ea65f688026cb95103795d0c51
This commit is contained in:
David Gillingham 2013-04-23 11:10:44 -05:00
parent 36ba9558a5
commit ee77d2ab1b
2 changed files with 70 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/IFPWE.java
View file

@ -58,6 +58,7 @@ import com.raytheon.uf.common.dataplugin.gfe.slice.IGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.slice.ScalarGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.slice.VectorGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.slice.WeatherGridSlice;
import com.raytheon.uf.common.dataplugin.gfe.type.Pair;
import com.raytheon.uf.common.dataplugin.gfe.util.GfeUtil;
import com.raytheon.uf.common.dataplugin.gfe.weather.WeatherKey;
import com.raytheon.uf.common.message.WsId;
@ -75,6 +76,7 @@ import com.raytheon.uf.common.time.TimeRange;
* May 7, 2008 njensen Initial creation
* Jan 22, 2010 4248 njensen Better error msgs
* Jul 25, 2012 #957 dgilling Implement getEditArea().
* Apr 23, 2013 #1937 dgilling Implement get().
*
* </pre>
*
@ -201,6 +203,37 @@ public class IFPWE {
return slice;
}
public List<Pair<IGridSlice, List<GridDataHistory>>> get(
List<TimeRange> times, boolean histories) {
GetGridRequest ggr = new GetGridRequest(parmId, times);
ServerResponse<List<IGridSlice>> sr = GridParmManager
.getGridData(Arrays.asList(ggr));
if (!sr.isOkay()) {
String msg = "Could not retrieve grid data for parm [" + parmId
+ "] for times " + times + ": " + sr.message();
statusHandler.error(msg);
return Collections.emptyList();
}
List<IGridSlice> data = sr.getPayload();
List<Pair<IGridSlice, List<GridDataHistory>>> rval = new ArrayList<Pair<IGridSlice, List<GridDataHistory>>>(
data.size());
for (IGridSlice grid : data) {
List<GridDataHistory> hists = Collections.emptyList();
if (histories) {
GridDataHistory[] h = grid.getHistory();
hists = new ArrayList<GridDataHistory>(h.length);
for (GridDataHistory entry : h) {
hists.add(entry);
}
}
rval.add(new Pair<IGridSlice, List<GridDataHistory>>(grid, hists));
}
return rval;
}
private void setItem(TimeRange time, IGridSlice gridSlice,
List<GridDataHistory> gdh) throws GfeException {
GFERecord rec = new GFERecord(parmId, time);

54
edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/ifpnetCDF.py
View file

@ -20,6 +20,7 @@
import string, getopt, sys, time, gzip, os, LogStream, stat, traceback
from collections import OrderedDict
import numpy
#import pupynere as NetCDF
try:
@ -61,12 +62,15 @@ from com.raytheon.uf.common.localization import LocalizationContext_Localization
# ------------ ---------- ----------- --------------------------
# 07/06/09 1995 bphillip Initial Creation.
# 03/11/13 1759 dgilling Removed unneeded methods.
# 04/23/13 1937 dgilling Reimplement WECache to match
# A1, big perf improvement.
#
#
#
BATCH_WRITE_COUNT = 10
# Original A1 BATCH WRITE COUNT was 10, we found doubling that
# lead to a significant performance increase.
BATCH_WRITE_COUNT = 20
BATCH_DELAY = 0.0
ifpNetcdfLogger=None
@ -95,44 +99,60 @@ def logDebug(*msg):
class WECache(object):
def __init__(self, we, inv):
self._we = we
self._grids = []
self._hist = []
self._inv = []
lst = []
self._inv = OrderedDict()
lst = list(inv)
while len(lst):
i = lst[:BATCH_WRITE_COUNT]
self._inv = self._inv + list(i)
javaTRs = ArrayList()
for tr in i:
javaTRs.add(iscUtil.toJavaTimeRange(tr))
gridsAndHist = self._we.get(javaTRs, True)
for idx, tr in enumerate(i):
pair = gridsAndHist.get(idx)
g = self.__encodeGridSlice(pair.getFirst())
h = self.__encodeGridHistory(pair.getSecond())
self._inv[tr] = (g, h)
lst = lst[BATCH_WRITE_COUNT:]
time.sleep(BATCH_DELAY)
def keys(self):
return tuple(self._inv)
return tuple(self._inv.keys())
def __getitem__(self, key):
grid = self._we.getItem(iscUtil.toJavaTimeRange(key))
history = grid.getGridDataHistory()
hist = []
for i in range(0, history.size()):
hist.append(history.get(i).getCodedString())
try:
return self._inv[key]
except KeyError:
grid = self._we.getItem(iscUtil.toJavaTimeRange(key))
pyGrid = self.__encodeGridSlice(grid)
history = grid.getGridDataHistory()
pyHist = self.__encodeGridHistory(history)
return (pyGrid, pyHist)
def __encodeGridSlice(self, grid):
gridType = grid.getGridInfo().getGridType().toString()
if gridType == "SCALAR":
return (grid.__numpy__[0], hist)
return grid.__numpy__[0]
elif gridType == "VECTOR":
vecGrids = grid.__numpy__
return ((vecGrids[0], vecGrids[1]), hist)
return (vecGrids[0], vecGrids[1])
elif gridType == "WEATHER":
keys = grid.getKeys()
keyList = []
for theKey in keys:
keyList.append(theKey.toString())
return ((grid.__numpy__[0],keyList), hist)
return (grid.__numpy__[0], keyList)
elif gridType =="DISCRETE":
keys = grid.getKey()
keyList = []
for theKey in keys:
keyList.append(theKey.toString())
return ((grid.__numpy__[0],keyList), hist)
return (grid.__numpy__[0], keyList)
def __encodeGridHistory(self, histories):
retVal = []
for i in xrange(histories.size()):
retVal.append(histories.get(i).getCodedString())
return tuple(retVal)
###-------------------------------------------------------------------------###