Issue #1954 Decompress ffmp geometries to save time loading them.

Change-Id: Id1de658c4a2cac0d30f8795ac0e84418deee1d92

Former-commit-id: 006c86d283 [formerly 3738e1cf81] [formerly 044fcc3cdb] [formerly 07bad53bc5 [formerly 044fcc3cdb [formerly b507a011fa222b5b2bf7662276fa29d5512f8e66]]]
Former-commit-id: 07bad53bc5
Former-commit-id: 02e3ec1402abaf320f1b434b2b8c4d22d7dcc7fe [formerly 13ab106ee5]
Former-commit-id: 2814ef6860

edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/HucLevelGeometriesFactory.java

@@ -19,16 +19,21 @@
  **/
 package com.raytheon.uf.common.dataplugin.ffmp;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.InputStream;
 import java.lang.ref.SoftReference;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.zip.GZIPInputStream;
 
 import com.raytheon.uf.common.localization.IPathManager;
 import com.raytheon.uf.common.localization.LocalizationContext;
@@ -38,6 +43,8 @@ import com.raytheon.uf.common.localization.LocalizationFile;
 import com.raytheon.uf.common.localization.PathManagerFactory;
 import com.raytheon.uf.common.localization.exception.LocalizationOpFailedException;
 import com.raytheon.uf.common.serialization.SerializationUtil;
+import com.raytheon.uf.common.serialization.adapters.FloatWKBReader;
+import com.raytheon.uf.common.serialization.adapters.FloatWKBWriter;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
@@ -50,7 +57,10 @@ import com.vividsolutions.jts.geom.Polygon;
 import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier;
 
 /**
- * TODO Add Description
+ * Manage a cache of geometries and envelopes for different areas/resolutions.
+ * The first time FFMP is loaded the geometries will be simplified and stored to
+ * localization for faster retrieval. All geometries and envelopes are held in
+ * memory by a soft reference or until they are explicitly cleared.
  * 
  * <pre>
  * 
@@ -59,6 +69,8 @@ import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier;
  * Date         Ticket#    Engineer    Description
  * ------------ ---------- ----------- --------------------------
  * Dec 9, 2010            rjpeter     Initial creation
+ * Apr 25, 2013 1954       bsteffen    Decompress ffmp geometries to save time
+ *                                     loading them.
  * 
  * </pre>
  * 
@@ -122,9 +134,44 @@ public class HucLevelGeometriesFactory {
 
             if (f.exists()) {
                 try {
-                    map = (Map<Long, Geometry>) SerializationUtil
-                            .transformFromThrift(FileUtil.file2bytes(
-                                    f.getFile(), true));
+                    File file = f.getFile();
+                    byte[] bytes = FileUtil.file2bytes(file, false);
+                    if (bytes[0] == (byte) 0x1f && bytes[1] == (byte) 0x8b) {
+                        // GZIP magic number is present, before 13.4.1 these
+                        // files were compressed and stored in a different
+                        // format, to maintain backwards compatibility we check
+                        // for compression and deserialize the old way. This
+                        // code can be removed any time after 13.5.1.
+                        System.out.println("Decompressing geometry files.");
+                        InputStream is = new ByteArrayInputStream(bytes);
+                        is = new GZIPInputStream(is, bytes.length);
+                        ByteArrayOutputStream os = new ByteArrayOutputStream(
+                                bytes.length * 3 / 2);
+                        byte[] buffer = new byte[1024 * 8];
+                        int numRead = 0;
+                        while ((numRead = is.read(buffer)) >= 0) {
+                            os.write(buffer, 0, numRead);
+                        }
+                        bytes = os.toByteArray();
+                        map = (Map<Long, Geometry>) SerializationUtil
+                                .transformFromThrift(Map.class, bytes);
+                        // save them back the new way.
+                        persistGeometryMap(dataKey, cwa, huc, map);
+                    } else {
+                        Map<Long, byte[]> serializableMap = (Map<Long, byte[]>) SerializationUtil
+                                .transformFromThrift(Map.class, bytes);
+                        FloatWKBReader reader = new FloatWKBReader(
+                                new GeometryFactory());
+                        map = new HashMap<Long, Geometry>(
+                                serializableMap.size());
+                        for (Entry<Long, byte[]> entry : serializableMap
+                                .entrySet()) {
+                            InputStream in = new ByteArrayInputStream(
+                                    entry.getValue());
+                            Geometry geom = reader.readGeometry(in);
+                            map.put(entry.getKey(), geom);
+                        }
+                    }
                     int sizeGuess = Math.max(
                             Math.abs(pfafs.size() - map.size()), 10);
                     pfafsToGenerate = new ArrayList<Long>(sizeGuess);
@@ -341,13 +388,23 @@ public class HucLevelGeometriesFactory {
 
     protected synchronized void persistGeometryMap(String dataKey, String cwa,
             String huc, Map<Long, Geometry> map) throws Exception {
+
         LocalizationContext lc = pathManager.getContext(
                 LocalizationType.COMMON_STATIC, LocalizationLevel.SITE);
         LocalizationFile lf = pathManager.getLocalizationFile(lc,
                 getGeomPath(dataKey, cwa, huc));
-        FileUtil.bytes2File(SerializationUtil.transformToThrift(map),
-                lf.getFile(), true);
+        FloatWKBWriter writer = new FloatWKBWriter();
+        ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
+        Map<Long, byte[]> serializableMap = new HashMap<Long, byte[]>();
+        for (Entry<Long, Geometry> entry : map.entrySet()) {
+            writer.writeGeometry(entry.getValue(), bos);
+            serializableMap.put(entry.getKey(), bos.toByteArray());
+            bos.reset();
+        }
+        byte[] bytes = SerializationUtil.transformToThrift(serializableMap);
+        FileUtil.bytes2File(bytes, lf.getFile(), false);
         lf.save();
+
     }
 
     protected synchronized String getGeomPath(String dataKey, String cwa,

edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/adapters/FloatWKBReader.java

@@ -0,0 +1,174 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/diclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.serialization.adapters;
+
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.Array;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
+import com.vividsolutions.jts.geom.GeometryCollection;
+import com.vividsolutions.jts.geom.GeometryFactory;
+import com.vividsolutions.jts.geom.LineString;
+import com.vividsolutions.jts.geom.LinearRing;
+import com.vividsolutions.jts.geom.MultiLineString;
+import com.vividsolutions.jts.geom.MultiPoint;
+import com.vividsolutions.jts.geom.MultiPolygon;
+import com.vividsolutions.jts.geom.Point;
+import com.vividsolutions.jts.geom.Polygon;
+import com.vividsolutions.jts.io.WKBConstants;
+
+/**
+ * Class for deserializing geometries in FloatWKB format. FloatWKB is a format
+ * based off of but completely incompatible with WKB. The only difference from
+ * WKB is that instead of reading every coordinate as a 8 byte double it is read
+ * as a 4 byte float. This cuts the size of objects in half while decreasing the
+ * precision.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 26, 2013 1954       bsteffen     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+public class FloatWKBReader {
+
+    private final GeometryFactory factory;
+
+    public FloatWKBReader(GeometryFactory factory) {
+        this.factory = factory;
+    }
+
+    public Geometry readGeometry(InputStream in) throws IOException {
+        return readGeometry((DataInput) new DataInputStream(in));
+    }
+
+    private Geometry readGeometry(DataInput di) throws IOException {
+        byte byteOrder = di.readByte();
+        if (byteOrder == WKBConstants.wkbNDR) {
+            throw new IOException(
+                    "WKBF does not currently support little endian");
+        }
+        int type = di.readInt();
+        switch (type) {
+        case WKBConstants.wkbPoint:
+            return readPoint(di);
+        case WKBConstants.wkbLineString:
+            return readLineString(di);
+        case WKBConstants.wkbPolygon:
+            return readPolygon(di);
+        case WKBConstants.wkbMultiPoint:
+            return readMultiPoint(di);
+        case WKBConstants.wkbMultiLineString:
+            return readMultiLineString(di);
+        case WKBConstants.wkbMultiPolygon:
+            return readMultiPolygon(di);
+        case WKBConstants.wkbGeometryCollection:
+            return readGeometryCollection(di);
+        }
+        // If the geometry contains three ordinates or a SRID it will also end
+        // up here
+        throw new IOException("Unknown WKB type " + (type & 0xff));
+    }
+
+    private Point readPoint(DataInput di) throws IOException {
+        return factory.createPoint(readCoordinate(di));
+    }
+
+    private LineString readLineString(DataInput di) throws IOException {
+        return factory.createLineString(readCoordinates(di));
+    }
+
+    private Polygon readPolygon(DataInput di) throws IOException {
+        int size = di.readInt();
+        LinearRing shell = null;
+        LinearRing[] holes = null;
+        shell = factory.createLinearRing(readCoordinates(di));
+        if (size > 1) {
+            holes = new LinearRing[size - 1];
+            for (int i = 1; i < size; i += 1) {
+                holes[i - 1] = factory.createLinearRing(readCoordinates(di));
+            }
+        }
+        return factory.createPolygon(shell, holes);
+    }
+
+    private MultiPoint readMultiPoint(DataInput di) throws IOException {
+        return factory.createMultiPoint(readMultiGeometry(di, Point.class));
+    }
+
+    private MultiLineString readMultiLineString(DataInput di)
+            throws IOException {
+        return factory.createMultiLineString(readMultiGeometry(di,
+                LineString.class));
+    }
+
+    private MultiPolygon readMultiPolygon(DataInput di) throws IOException {
+        return factory.createMultiPolygon(readMultiGeometry(di, Polygon.class));
+    }
+
+    private GeometryCollection readGeometryCollection(DataInput di)
+            throws IOException {
+        return factory.createGeometryCollection(readMultiGeometry(di,
+                Geometry.class));
+    }
+
+    private <T extends Geometry> T[] readMultiGeometry(DataInput di,
+            Class<T> geomType) throws IOException {
+        int size = di.readInt();
+        @SuppressWarnings("unchecked")
+        T[] geoms = (T[]) Array.newInstance(geomType, size);
+        for (int i = 0; i < size; i++) {
+            Geometry g = readGeometry(di);
+            if (geomType.isInstance(g)) {
+                geoms[i] = geomType.cast(g);
+            } else {
+                throw new IOException("Expected a "
+                        + geomType.getClass().getSimpleName()
+                        + " but recieved a " + g.getClass().getSimpleName());
+            }
+        }
+        return geoms;
+    }
+
+    private Coordinate[] readCoordinates(DataInput di) throws IOException {
+        int size = di.readInt();
+        Coordinate[] coordinates = new Coordinate[size];
+        for (int i = 0; i < size; i += 1) {
+            coordinates[i] = readCoordinate(di);
+        }
+        return coordinates;
+    }
+
+    private Coordinate readCoordinate(DataInput di) throws IOException {
+        return new Coordinate(di.readFloat(), di.readFloat());
+    }
+
+}

edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/adapters/FloatWKBWriter.java

@@ -0,0 +1,152 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/diclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.serialization.adapters;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
+import com.vividsolutions.jts.geom.GeometryCollection;
+import com.vividsolutions.jts.geom.LineString;
+import com.vividsolutions.jts.geom.MultiLineString;
+import com.vividsolutions.jts.geom.MultiPoint;
+import com.vividsolutions.jts.geom.MultiPolygon;
+import com.vividsolutions.jts.geom.Point;
+import com.vividsolutions.jts.geom.Polygon;
+import com.vividsolutions.jts.io.WKBConstants;
+
+/**
+ * Class for serializing geometries in FloatWKB format. FloatWKB is a format
+ * based off of but completely incompatible with WKB. The only difference from
+ * WKB is that instead of writing every coordinate as a 8 byte double it is
+ * written as a 4 byte float. This cuts the size of objects in half while
+ * decreasing the precision.
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 26, 2013 1954       bsteffen     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author bsteffen
+ * @version 1.0
+ */
+public class FloatWKBWriter {
+
+    public void writeGeometry(Geometry geom, OutputStream out)
+            throws IOException {
+        writeGeometry(geom, (DataOutput) new DataOutputStream(out));
+    }
+
+    private void writeGeometry(Geometry geom, DataOutput d)
+            throws IOException {
+        d.write(WKBConstants.wkbXDR);
+        if (geom instanceof Point)
+            writePoint((Point) geom, d);
+        else if (geom instanceof LineString)
+            writeLineString((LineString) geom, d);
+        else if (geom instanceof Polygon)
+            writePolygon((Polygon) geom, d);
+        else if (geom instanceof MultiPoint)
+            writeMultiPoint((MultiPoint) geom, d);
+        else if (geom instanceof MultiLineString)
+            writeMultiLineString((MultiLineString) geom, d);
+        else if (geom instanceof MultiPolygon)
+            writeMultiPolygon((MultiPolygon) geom, d);
+        else if (geom instanceof GeometryCollection)
+            writeGeometryCollection((GeometryCollection) geom, d);
+        else {
+            throw new IOException("Unknown Geometry type: "
+                    + geom.getClass().getSimpleName());
+        }
+    }
+
+    private void writePoint(Point p, DataOutput d) throws IOException {
+        d.writeInt(WKBConstants.wkbPoint);
+        writeCoordinate(p.getCoordinate(), d);
+    }
+
+    private void writeLineString(LineString ls, DataOutput d)
+            throws IOException {
+        d.writeInt(WKBConstants.wkbLineString);
+        writeCoordinates(ls.getCoordinates(), d);
+    }
+
+    private void writePolygon(Polygon polygon, DataOutput d) throws IOException {
+        d.writeInt(WKBConstants.wkbPolygon);
+        d.writeInt(polygon.getNumInteriorRing() + 1);
+        writeCoordinates(polygon.getExteriorRing().getCoordinates(), d);
+        for (int i = 0; i < polygon.getNumInteriorRing(); i += 1) {
+            writeCoordinates(polygon.getInteriorRingN(i).getCoordinates(), d);
+        }
+    }
+
+    private void writeMultiPoint(MultiPoint mp, DataOutput d)
+            throws IOException {
+        d.writeInt(WKBConstants.wkbMultiPoint);
+        writeMultiGeometry(mp, d);
+    }
+
+    private void writeMultiLineString(MultiLineString mls, DataOutput d)
+            throws IOException {
+        d.writeInt(WKBConstants.wkbMultiLineString);
+        writeMultiGeometry(mls, d);
+    }
+
+    private void writeMultiPolygon(MultiPolygon mp, DataOutput d)
+            throws IOException {
+        d.writeInt(WKBConstants.wkbMultiPolygon);
+        writeMultiGeometry(mp, d);
+    }
+
+    private void writeGeometryCollection(GeometryCollection gc, DataOutput d)
+            throws IOException {
+        d.writeInt(WKBConstants.wkbGeometryCollection);
+        writeMultiGeometry(gc, d);
+    }
+
+    private void writeMultiGeometry(GeometryCollection gc, DataOutput d)
+            throws IOException {
+        d.writeInt(gc.getNumGeometries());
+        for (int i = 0; i < gc.getNumGeometries(); i++) {
+            writeGeometry(gc.getGeometryN(i), d);
+        }
+    }
+
+    private void writeCoordinates(Coordinate[] coordinates, DataOutput d)
+            throws IOException {
+        d.writeInt(coordinates.length);
+        for (Coordinate c : coordinates) {
+            writeCoordinate(c, d);
+        }
+    }
+
+    private void writeCoordinate(Coordinate c, DataOutput d) throws IOException {
+        d.writeFloat((float) c.x);
+        d.writeFloat((float) c.y);
+    }
+}