Issue #1941: Speed up iscMosaic by properly utilizing WECache.

Change-Id: Ic8d7ea31cf8f1a989c4f06c0a60c9a351ed8ce8e

Former-commit-id: 60bcd26d36 [formerly 60bcd26d36 [formerly f3d7241b3eafb86046cdcc6c28197244b55f21e9]]
Former-commit-id: 159302cf8c
Former-commit-id: 14701c14d7

edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/smartinit/IFPWE.java

@@ -24,7 +24,9 @@ import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collections;
 import java.util.Date;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map.Entry;
 import java.util.TimeZone;
 
 import com.raytheon.edex.plugin.gfe.config.IFPServerConfig;
@@ -35,7 +37,6 @@ import com.raytheon.edex.plugin.gfe.server.lock.LockManager;
 import com.raytheon.edex.plugin.gfe.util.SendNotifications;
 import com.raytheon.uf.common.dataplugin.gfe.GridDataHistory;
 import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord;
-import com.raytheon.uf.common.dataplugin.gfe.db.objects.GFERecord.GridType;
 import com.raytheon.uf.common.dataplugin.gfe.db.objects.GridParmInfo;
 import com.raytheon.uf.common.dataplugin.gfe.db.objects.ParmID;
 import com.raytheon.uf.common.dataplugin.gfe.discrete.DiscreteKey;
@@ -77,6 +78,9 @@ import com.raytheon.uf.common.time.TimeRange;
  * Jan 22, 2010  4248       njensen     Better error msgs
  * Jul 25, 2012  #957       dgilling    Implement getEditArea().
  * Apr 23, 2013  #1937      dgilling    Implement get().
+ * Apr 23, 2013  #1941      dgilling    Implement put(), add methods to build
+ *                                      Scalar/VectorGridSlices, refactor
+ *                                      Discrete/WeatherGridSlices builders.
  * 
  * </pre>
  * 
@@ -234,6 +238,76 @@ public class IFPWE {
         return rval;
     }
 
+    /**
+     * Stores the provided grid slices into this weather element's permanent
+     * storage.
+     * 
+     * @param inventory
+     *            A Map of TimeRanges to IGridSlices to be saved. Time is the
+     *            slice's valid time.
+     * @param timeRangeSpan
+     *            The replacement time range of grids to be saved. Must cover
+     *            each individual TimeRange in inventory.
+     * @throws GfeException
+     *             If an error occurs while trying to obtain a lock on the
+     *             destination database.
+     */
+    public void put(LinkedHashMap<TimeRange, IGridSlice> inventory,
+            TimeRange timeRangeSpan) throws GfeException {
+        statusHandler.debug("Getting lock for ParmID: " + parmId + " TR: "
+                + timeRangeSpan);
+        ServerResponse<List<LockTable>> lockResponse = LockManager
+                .getInstance().requestLockChange(
+                        new LockRequest(parmId, timeRangeSpan, LockMode.LOCK),
+                        wsId, siteId);
+        if (lockResponse.isOkay()) {
+            statusHandler.debug("LOCKING: Lock granted for: " + wsId
+                    + " for time range: " + timeRangeSpan);
+        } else {
+            statusHandler.error("Could not lock TimeRange " + timeRangeSpan
+                    + " for parm [" + parmId + "]: " + lockResponse.message());
+            throw new GfeException("Request lock failed. "
+                    + lockResponse.message());
+        }
+
+        List<GFERecord> records = new ArrayList<GFERecord>(inventory.size());
+        for (Entry<TimeRange, IGridSlice> entry : inventory.entrySet()) {
+            GFERecord rec = new GFERecord(parmId, entry.getKey());
+            rec.setGridHistory(entry.getValue().getHistory());
+            rec.setMessageData(entry.getValue());
+            records.add(rec);
+        }
+        SaveGridRequest sgr = new SaveGridRequest(parmId, timeRangeSpan,
+                records);
+
+        try {
+            ServerResponse<?> sr = GridParmManager.saveGridData(
+                    Arrays.asList(sgr), wsId, siteId);
+            if (sr.isOkay()) {
+                SendNotifications.send(sr.getNotifications());
+            } else {
+                statusHandler.error("Unable to save grids for parm [" + parmId
+                        + "] over time range " + timeRangeSpan + ": "
+                        + sr.message());
+            }
+        } finally {
+            ServerResponse<List<LockTable>> unLockResponse = LockManager
+                    .getInstance().requestLockChange(
+                            new LockRequest(parmId, timeRangeSpan,
+                                    LockMode.UNLOCK), wsId, siteId);
+            if (unLockResponse.isOkay()) {
+                statusHandler.debug("LOCKING: Unlocked for: " + wsId + " TR: "
+                        + timeRangeSpan);
+            } else {
+                statusHandler.error("Could not unlock TimeRange "
+                        + timeRangeSpan + " for parm [" + parmId + "]: "
+                        + lockResponse.message());
+                throw new GfeException("Request unlock failed. "
+                        + unLockResponse.message());
+            }
+        }
+    }
+
     private void setItem(TimeRange time, IGridSlice gridSlice,
             List<GridDataHistory> gdh) throws GfeException {
         GFERecord rec = new GFERecord(parmId, time);
@@ -373,9 +447,7 @@ public class IFPWE {
     public void setItemDiscrete(TimeRange time, byte[] discreteData,
             String keys, List<GridDataHistory> gdhList) throws GfeException {
         IGridSlice gridSlice = buildDiscreteSlice(time, discreteData, keys,
-                gpi.getGridType());
+                gdhList);
-        gridSlice
-                .setHistory(gdhList.toArray(new GridDataHistory[gdhList.size()]));
         setItem(time, gridSlice, gdhList);
     }
 
@@ -393,9 +465,7 @@ public class IFPWE {
     public void setItemWeather(TimeRange time, byte[] weatherData, String keys,
             List<GridDataHistory> gdhList) throws GfeException {
         IGridSlice gridSlice = buildWeatherSlice(time, weatherData, keys,
-                gpi.getGridType());
+                gdhList);
-        gridSlice
-                .setHistory(gdhList.toArray(new GridDataHistory[gdhList.size()]));
         setItem(time, gridSlice, gdhList);
     }
 
@@ -433,60 +503,96 @@ public class IFPWE {
         return keys.toArray(new String[keys.size()]);
     }
 
+    /**
+     * Builds a ScalarGridSlice to store.
+     * 
+     * @param time
+     *            The valid time of the slice.
+     * @param data
+     *            A float array that corresponds to the slice's data.
+     * @param history
+     *            The GridDataHistory for the new slice.
+     * @return A ScalarGridSlice based on the provided data, valid for the given
+     *         time, with the provided history.
+     */
+    public ScalarGridSlice buildScalarSlice(TimeRange time, float[] data,
+            List<GridDataHistory> history) {
+        return new ScalarGridSlice(time, gpi, history, new Grid2DFloat(gpi
+                .getGridLoc().getNx(), gpi.getGridLoc().getNy(), data));
+    }
+
+    /**
+     * Builds a VectorGridSlice to store.
+     * 
+     * @param time
+     *            The valid time of the slice.
+     * @param magData
+     *            A float array that corresponds to the slice's magnitude data.
+     * @param dirData
+     *            A float array that corresponds to the slice's directional
+     *            data.
+     * @param history
+     *            The GridDataHistory for the new slice.
+     * @return A VectorGridSlice based on the provided data, valid for the given
+     *         time, with the provided history.
+     */
+    public VectorGridSlice buildVectorSlice(TimeRange time, float[] magData,
+            float[] dirData, List<GridDataHistory> history) {
+        return new VectorGridSlice(time, gpi, history, new Grid2DFloat(gpi
+                .getGridLoc().getNx(), gpi.getGridLoc().getNy(), magData),
+                new Grid2DFloat(gpi.getGridLoc().getNx(), gpi.getGridLoc()
+                        .getNy(), dirData));
+    }
+
     /**
      * Builds a discrete grid slice to store
      * 
      * @param time
-     *            the time of the data
+     *            The valid time of the data.
-     * @param slice
+     * @param bytes
-     *            an Object[] { byte[], String } corresponding to discrete/wx
+     *            A byte[] corresponding to discrete
-     *            types
+     * @param keyString
-     * @param type
+     *            Python encoded form of discrete keys.
-     *            the type of the data
+     * @param history
+     *            histories for this grid.
      * @return
-     * @throws GfeException
      */
-    private IGridSlice buildDiscreteSlice(TimeRange time, byte[] bytes,
+    public DiscreteGridSlice buildDiscreteSlice(TimeRange time, byte[] bytes,
-            String keyString, GridType type) throws GfeException {
+            String keyString, List<GridDataHistory> history) {
         List<DiscreteKey> discreteKeyList = new ArrayList<DiscreteKey>();
         List<String> keys = GfeUtil.discreteKeyStringToList(keyString);
 
         for (String k : keys) {
             discreteKeyList.add(new DiscreteKey(siteId, k, parmId));
         }
-        return new DiscreteGridSlice(
+        return new DiscreteGridSlice(time, gpi, history, new Grid2DByte(gpi
-                time,
+                .getGridLoc().getNx(), gpi.getGridLoc().getNy(), bytes),
-                gpi,
+                discreteKeyList);
-                new GridDataHistory[] {},
-                new Grid2DByte(gpi.getGridLoc().getNx(), gpi.getGridLoc()
-                        .getNy(), bytes),
-                discreteKeyList.toArray(new DiscreteKey[discreteKeyList.size()]));
     }
 
     /**
      * Builds a weather grid slice to store
      * 
      * @param time
-     *            the time of the data
+     *            The valid time of the data.
-     * @param slice
+     * @param bytes
-     *            an Object[] { byte[], String } corresponding to weather/wx
+     *            A byte[] corresponding to weather
-     *            types
+     * @param keyString
-     * @param type
+     *            Python encoded form of weather keys.
-     *            the type of the data
+     * @param history
+     *            histories for this grid.
      * @return
-     * @throws GfeException
      */
-    private IGridSlice buildWeatherSlice(TimeRange time, byte[] bytes,
+    public WeatherGridSlice buildWeatherSlice(TimeRange time, byte[] bytes,
-            String keyString, GridType type) throws GfeException {
+            String keyString, List<GridDataHistory> history) {
         List<WeatherKey> weatherKeyList = new ArrayList<WeatherKey>();
         List<String> keys = GfeUtil.discreteKeyStringToList(keyString);
         for (String k : keys) {
             weatherKeyList.add(new WeatherKey(siteId, k));
         }
-        return new WeatherGridSlice(time, gpi, new GridDataHistory[] {},
+        return new WeatherGridSlice(time, gpi, history, new Grid2DByte(gpi
-                new Grid2DByte(gpi.getGridLoc().getNx(), gpi.getGridLoc()
+                .getGridLoc().getNx(), gpi.getGridLoc().getNy(), bytes),
-                        .getNy(), bytes),
+                weatherKeyList);
-                weatherKeyList.toArray(new WeatherKey[weatherKeyList.size()]));
     }
 
     @Override

edexOsgi/com.raytheon.edex.plugin.gfe/utility/edex_static/base/gfe/isc/iscMosaic.py

@@ -32,6 +32,7 @@ import numpy
 import JUtil
 
 from java.util import ArrayList
+from java.util import LinkedHashMap
 from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DFloat
 from com.raytheon.uf.common.dataplugin.gfe.grid import Grid2DByte
 from com.raytheon.uf.common.time import TimeRange
@@ -74,7 +75,8 @@ from com.raytheon.uf.edex.database.cluster import ClusterTask
 #    07/06/09        1995          bphillip       Initial Creation.
 #    01/17/13        15588         jdynina        Fixed Publish history removal
 #    03/12/13        1759          dgilling       Remove unnecessary command line
-#                                                 processing. 
+#                                                 processing.
+#    04/24/13        1941          dgilling       Re-port WECache to match A1.
 # 
 # 
 
@@ -86,53 +88,49 @@ ISC_USER="isc"
 
 class WECache(object): 
     def __init__(self, we, tr=None):
-        self._grids = []
-        self._hist = []
         self._we = we
-        self._inv = []
+        self._inv = {}
-        theKeys = self._we.getKeys() 
+        self._invCache = None
-        
-        for i in range(0, theKeys.size()):
-            self._inv.append(iscUtil.transformTime(theKeys.get(i)))
         
+        javaInv = self._we.getKeys()
+        pyInv = []
+        for i in xrange(javaInv.size()):
+            pyInv.append(iscUtil.transformTime(javaInv.get(i)))
 
         # Dont get grids outside of the passed in timerange.
         if tr:
             tokill = []
-            for i, t in enumerate(self._inv):
+            for i, t in enumerate(pyInv):
                 if not self.overlaps(tr, t):
                     tokill.append(i)
             tokill.reverse()
             for i in tokill:
-                del self._inv[i]
+                del pyInv[i]
+
+        javaTRs = ArrayList()
+        for tr in pyInv:
+            javaTRs.add(iscUtil.toJavaTimeRange(tr))
+        gridsAndHist = self._we.get(javaTRs, True)
+        for idx, tr in enumerate(pyInv):
+            pair = gridsAndHist.get(idx)
+            g = self.__encodeGridSlice(pair.getFirst())
+            h = self.__encodeGridHistory(pair.getSecond())
+            self._inv[tr] = (g, h)
 
     def keys(self):
-        return tuple(self._inv)
+        if not self._invCache:
+            self._invCache = tuple(sorted(self._inv.keys(), key=lambda t: t[0]))
+        return self._invCache
 
     def __getitem__(self, key):
-        grid = self._we.getItem(iscUtil.toJavaTimeRange(key))
+        try:
-        history = grid.getGridDataHistory()
+            return self._inv[key]
-        hist = []
+        except KeyError:
-        for i in range(0, history.size()):
+            grid = self._we.getItem(iscUtil.toJavaTimeRange(key))
-            hist.append(history.get(i))
+            pyGrid = self.__encodeGridSlice(grid)
-        gridType = grid.getGridInfo().getGridType().toString()
+            history = grid.getGridDataHistory()
-        if gridType == "SCALAR":
+            pyHist = self.__encodeGridHistory(history)
-            return (grid.__numpy__[0], hist)
+            return (pyGrid, pyHist)
-        elif gridType == "VECTOR":
-            vecGrids = grid.__numpy__
-            return ((vecGrids[0], vecGrids[1]), hist)
-        elif gridType == "WEATHER":
-            keys = grid.getKeys()
-            keyList = []
-            for theKey in keys:
-                keyList.append(theKey.toString())
-            return ((grid.__numpy__[0], keyList), hist)
-        elif gridType == "DISCRETE":
-            keys = grid.getKey()
-            keyList = []
-            for theKey in keys:
-                keyList.append(theKey.toString())
-            return ((grid.__numpy__[0], keyList), hist)
 
     def __setitem__(self, tr, value):
         if value is None:
@@ -142,48 +140,106 @@ class WECache(object):
         
         # Remove any overlapping grids
         tokill = []
-        for i, itr in enumerate(self._inv):
+        for itr in self._inv:
             if self.overlaps(tr, itr):
-                tokill.append(i)
+                tokill.append(itr)
-        tokill.reverse()
         for i in tokill:
-            del self._inv[i] 
+            del self._inv[i]
+            self._invCache = None
         
         # Now add the new grid if it exists
         if grid is not None:
-            timeRange=iscUtil.toJavaTimeRange(tr)
+            self._inv[tr] = (grid, hist)
-            LogStream.logDebug("iscMosaic: Saving Parm:",self._we.getParmid(),"TR:",timeRange)
+            self._invCache = None            
-            gridType = self._we.getGridType()
-            index = bisect.bisect_left(map(lambda x : x[0], self._inv), tr[0])
-            self._inv.insert(index, tr)   
-            history = ArrayList()
 
-            for h in hist:
+    def flush(self):
-		dbName = self._we.getParmid().getDbId().toString()
+        """Actually writes the contents of the WECache to HDF5/DB"""
-		if dbName.find('Fcst') != -1:
+        # get cache inventory in time range order
-                    #strip out publish time to allow for publishing correctly
+        # we want to write to disk in contiguous time range blocks so we only
-                    #when merging Fcst out of A1 
+        # overwrite what we have full sets of grids for.        
-                    hh = GridDataHistory(h)
+        inv = list(self.keys())
-                    hh.setPublishTime(None)
+        # Don't believe the grid slices need to be in time order when saving
-                    history.add(hh)
+        # but leaving them that way just in case.
-		else:
+        gridsToSave = LinkedHashMap()
-		    history.add(GridDataHistory(h)) 
+        while inv:
-            if gridType == 'SCALAR':
+            # retrieve the next BATCH of grids to persist
-                self._we.setItemScalar(timeRange, grid.astype(numpy.float32), history)
+            i = inv[:BATCH_WRITE_COUNT]
-            elif gridType == 'VECTOR':
+            # pre-compute the replacement TR for the save requests generated by
-                self._we.setItemVector(timeRange, grid[0].astype(numpy.float32), grid[1].astype(numpy.float32), history)
+            # IFPWE.put().
-            elif gridType == 'WEATHER':
+            # since the inventory is in order it's the start time of the 
-                self._we.setItemWeather(timeRange, grid[0].astype(numpy.byte), str(grid[1]), history) 
+            # first TR and the end time of the last TR.
-            elif gridType == 'DISCRETE':
+            gridSaveTR = iscUtil.toJavaTimeRange((i[0][0], i[-1][1]))
-                self._we.setItemDiscrete(timeRange, grid[0].astype(numpy.byte), str(grid[1]), history)
+            for tr in i:
-            LogStream.logDebug("iscMosaic: Successfully saved Parm:",self._we.getParmid(),"Time:",timeRange)
+                javaTR = iscUtil.toJavaTimeRange(tr)
-                
+                pyGrid, pyHist = self._inv[tr]
+                javaHist = self.__buildJavaGridHistory(pyHist) 
+                javaGrid = self.__buildJavaGridSlice(javaTR, pyGrid, javaHist)
+                gridsToSave.put(javaTR, javaGrid)
+            self._we.put(gridsToSave, gridSaveTR)
+            # delete the persisted items from the cache and our copy of the
+            # inventory
+            gridsToSave.clear()
+            for tr in i:
+                del self._inv[tr]
+            self._invCache = None
+            inv = inv[BATCH_WRITE_COUNT:]
+            time.sleep(BATCH_DELAY)
+            
 
     def overlaps(self, tr1, tr2):
         if (tr1[0] >= tr2[0] and tr1[0] < tr2[1]) or \
            (tr2[0] >= tr1[0] and tr2[0] < tr1[1]): 
             return True
         return False
+    
+    def __encodeGridSlice(self, grid):
+        gridType = self._we.getGridType()
+        if gridType == "SCALAR":
+            return grid.__numpy__[0]
+        elif gridType == "VECTOR":
+            vecGrids = grid.__numpy__
+            return (vecGrids[0], vecGrids[1])
+        elif gridType == "WEATHER":
+            keys = grid.getKeys()
+            keyList = []
+            for theKey in keys:
+                keyList.append(theKey.toString())
+            return (grid.__numpy__[0], keyList)
+        elif gridType =="DISCRETE":
+            keys = grid.getKey()
+            keyList = []
+            for theKey in keys:
+                keyList.append(theKey.toString())
+            return (grid.__numpy__[0], keyList)
+    
+    def __encodeGridHistory(self, histories):
+        retVal = []
+        for i in xrange(histories.size()):
+            retVal.append(histories.get(i).getCodedString())
+        return tuple(retVal)
+    
+    def __buildJavaGridSlice(self, tr, grid, history):
+        gridType = self._we.getGridType()
+        if gridType == "SCALAR":
+            return self._we.buildScalarSlice(tr, grid.astype(numpy.float32), history)
+        elif gridType == "VECTOR":
+            return self._we.buildVectorSlice(tr, grid[0].astype(numpy.float32), grid[1].astype(numpy.float32), history)
+        elif gridType == "WEATHER":
+            return self._we.buildWeatherSlice(tr, grid[0].astype(numpy.byte), str(grid[1]), history)
+        elif gridType == "DISCRETE":
+            return self._we.buildDiscreteSlice(tr, grid[0].astype(numpy.byte), str(grid[1]), history)
+    
+    def __buildJavaGridHistory(self, histories):
+        retVal = ArrayList()
+        blankPubTime = "Fcst" in self._we.getParmid().getDbId().toString()
+        for histEntry in histories:
+            javaHist = GridDataHistory(histEntry)
+            # strip out publish time to allow for publishing correctly
+            # when merging Fcst out of A1 
+            if blankPubTime:
+                javaHist.setPublishTime(None)
+            retVal.add(javaHist)
+        return retVal
 
 
 class IscMosaic:
@@ -549,7 +605,8 @@ class IscMosaic:
                     # Returns tuple of (parmName, TR, #grids, #fails)
                     if len(inTimesProc):
                         totalTimeRange = (inTimesProc[0][0], inTimesProc[ -1][ -1] - 3600)
-                        
+                    self._wec.flush()
+                    
                     retryAttempt = retries
                 except:
                     retryAttempt = retryAttempt + 1