xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1954 Decompress ffmp geometries to save time loading them. Change-Id: Id1de658c4a2cac0d30f8795ac0e84418deee1d92" into omaha_13.4.1

Browse source 
Former-commit-id: 4848941215 [formerly 4848941215 [formerly 37f37675ce6fb9b238856517bea69258d6b3ea89]]
Former-commit-id: 5e88a9403c
Former-commit-id: dbdeb13113
This commit is contained in:
Nate Jensen 2013-04-26 14:56:06 -05:00 • committed by Gerrit Code Review
commit 4cccf47108
3 changed files with 389 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
edexOsgi/com.raytheon.uf.common.dataplugin.ffmp/src/com/raytheon/uf/common/dataplugin/ffmp/HucLevelGeometriesFactory.java
View file

@ -19,16 +19,21 @@
**/
package com.raytheon.uf.common.dataplugin.ffmp;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.InputStream;
import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.zip.GZIPInputStream;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
@ -38,6 +43,8 @@ import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.localization.exception.LocalizationOpFailedException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.serialization.adapters.FloatWKBReader;
import com.raytheon.uf.common.serialization.adapters.FloatWKBWriter;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@ -50,7 +57,10 @@ import com.vividsolutions.jts.geom.Polygon;
import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier;
/**
* TODO Add Description
* Manage a cache of geometries and envelopes for different areas/resolutions.
* The first time FFMP is loaded the geometries will be simplified and stored to
* localization for faster retrieval. All geometries and envelopes are held in
* memory by a soft reference or until they are explicitly cleared.
*
* <pre>
*
@ -59,6 +69,8 @@ import com.vividsolutions.jts.simplify.TopologyPreservingSimplifier;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 9, 2010 rjpeter Initial creation
* Apr 25, 2013 1954 bsteffen Decompress ffmp geometries to save time
* loading them.
*
* </pre>
*
@ -122,9 +134,44 @@ public class HucLevelGeometriesFactory {
if (f.exists()) {
try {
map = (Map<Long, Geometry>) SerializationUtil
.transformFromThrift(FileUtil.file2bytes(
f.getFile(), true));
File file = f.getFile();
byte[] bytes = FileUtil.file2bytes(file, false);
if (bytes[0] == (byte) 0x1f && bytes[1] == (byte) 0x8b) {
// GZIP magic number is present, before 13.4.1 these
// files were compressed and stored in a different
// format, to maintain backwards compatibility we check
// for compression and deserialize the old way. This
// code can be removed any time after 13.5.1.
System.out.println("Decompressing geometry files.");
InputStream is = new ByteArrayInputStream(bytes);
is = new GZIPInputStream(is, bytes.length);
ByteArrayOutputStream os = new ByteArrayOutputStream(
bytes.length * 3 / 2);
byte[] buffer = new byte[1024 * 8];
int numRead = 0;
while ((numRead = is.read(buffer)) >= 0) {
os.write(buffer, 0, numRead);
}
bytes = os.toByteArray();
map = (Map<Long, Geometry>) SerializationUtil
.transformFromThrift(Map.class, bytes);
// save them back the new way.
persistGeometryMap(dataKey, cwa, huc, map);
} else {
Map<Long, byte[]> serializableMap = (Map<Long, byte[]>) SerializationUtil
.transformFromThrift(Map.class, bytes);
FloatWKBReader reader = new FloatWKBReader(
new GeometryFactory());
map = new HashMap<Long, Geometry>(
serializableMap.size());
for (Entry<Long, byte[]> entry : serializableMap
.entrySet()) {
InputStream in = new ByteArrayInputStream(
entry.getValue());
Geometry geom = reader.readGeometry(in);
map.put(entry.getKey(), geom);
}
}
int sizeGuess = Math.max(
Math.abs(pfafs.size() - map.size()), 10);
pfafsToGenerate = new ArrayList<Long>(sizeGuess);
@ -341,13 +388,23 @@ public class HucLevelGeometriesFactory {
protected synchronized void persistGeometryMap(String dataKey, String cwa,
String huc, Map<Long, Geometry> map) throws Exception {
LocalizationContext lc = pathManager.getContext(
LocalizationType.COMMON_STATIC, LocalizationLevel.SITE);
LocalizationFile lf = pathManager.getLocalizationFile(lc,
getGeomPath(dataKey, cwa, huc));
FileUtil.bytes2File(SerializationUtil.transformToThrift(map),
lf.getFile(), true);
FloatWKBWriter writer = new FloatWKBWriter();
ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
Map<Long, byte[]> serializableMap = new HashMap<Long, byte[]>();
for (Entry<Long, Geometry> entry : map.entrySet()) {
writer.writeGeometry(entry.getValue(), bos);
serializableMap.put(entry.getKey(), bos.toByteArray());
bos.reset();
}
byte[] bytes = SerializationUtil.transformToThrift(serializableMap);
FileUtil.bytes2File(bytes, lf.getFile(), false);
lf.save();
}
protected synchronized String getGeomPath(String dataKey, String cwa,

174
edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/adapters/FloatWKBReader.java Normal file
View file

@ -0,0 +1,174 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/diclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.serialization.adapters;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.Array;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryCollection;
import com.vividsolutions.jts.geom.GeometryFactory;
import com.vividsolutions.jts.geom.LineString;
import com.vividsolutions.jts.geom.LinearRing;
import com.vividsolutions.jts.geom.MultiLineString;
import com.vividsolutions.jts.geom.MultiPoint;
import com.vividsolutions.jts.geom.MultiPolygon;
import com.vividsolutions.jts.geom.Point;
import com.vividsolutions.jts.geom.Polygon;
import com.vividsolutions.jts.io.WKBConstants;
/**
* Class for deserializing geometries in FloatWKB format. FloatWKB is a format
* based off of but completely incompatible with WKB. The only difference from
* WKB is that instead of reading every coordinate as a 8 byte double it is read
* as a 4 byte float. This cuts the size of objects in half while decreasing the
* precision.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 26, 2013 1954 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class FloatWKBReader {
private final GeometryFactory factory;
public FloatWKBReader(GeometryFactory factory) {
this.factory = factory;
}
public Geometry readGeometry(InputStream in) throws IOException {
return readGeometry((DataInput) new DataInputStream(in));
}
private Geometry readGeometry(DataInput di) throws IOException {
byte byteOrder = di.readByte();
if (byteOrder == WKBConstants.wkbNDR) {
throw new IOException(
"WKBF does not currently support little endian");
}
int type = di.readInt();
switch (type) {
case WKBConstants.wkbPoint:
return readPoint(di);
case WKBConstants.wkbLineString:
return readLineString(di);
case WKBConstants.wkbPolygon:
return readPolygon(di);
case WKBConstants.wkbMultiPoint:
return readMultiPoint(di);
case WKBConstants.wkbMultiLineString:
return readMultiLineString(di);
case WKBConstants.wkbMultiPolygon:
return readMultiPolygon(di);
case WKBConstants.wkbGeometryCollection:
return readGeometryCollection(di);
}
// If the geometry contains three ordinates or a SRID it will also end
// up here
throw new IOException("Unknown WKB type " + (type & 0xff));
}
private Point readPoint(DataInput di) throws IOException {
return factory.createPoint(readCoordinate(di));
}
private LineString readLineString(DataInput di) throws IOException {
return factory.createLineString(readCoordinates(di));
}
private Polygon readPolygon(DataInput di) throws IOException {
int size = di.readInt();
LinearRing shell = null;
LinearRing[] holes = null;
shell = factory.createLinearRing(readCoordinates(di));
if (size > 1) {
holes = new LinearRing[size - 1];
for (int i = 1; i < size; i += 1) {
holes[i - 1] = factory.createLinearRing(readCoordinates(di));
}
}
return factory.createPolygon(shell, holes);
}
private MultiPoint readMultiPoint(DataInput di) throws IOException {
return factory.createMultiPoint(readMultiGeometry(di, Point.class));
}
private MultiLineString readMultiLineString(DataInput di)
throws IOException {
return factory.createMultiLineString(readMultiGeometry(di,
LineString.class));
}
private MultiPolygon readMultiPolygon(DataInput di) throws IOException {
return factory.createMultiPolygon(readMultiGeometry(di, Polygon.class));
}
private GeometryCollection readGeometryCollection(DataInput di)
throws IOException {
return factory.createGeometryCollection(readMultiGeometry(di,
Geometry.class));
}
private <T extends Geometry> T[] readMultiGeometry(DataInput di,
Class<T> geomType) throws IOException {
int size = di.readInt();
@SuppressWarnings("unchecked")
T[] geoms = (T[]) Array.newInstance(geomType, size);
for (int i = 0; i < size; i++) {
Geometry g = readGeometry(di);
if (geomType.isInstance(g)) {
geoms[i] = geomType.cast(g);
} else {
throw new IOException("Expected a "
+ geomType.getClass().getSimpleName()
+ " but recieved a " + g.getClass().getSimpleName());
}
}
return geoms;
}
private Coordinate[] readCoordinates(DataInput di) throws IOException {
int size = di.readInt();
Coordinate[] coordinates = new Coordinate[size];
for (int i = 0; i < size; i += 1) {
coordinates[i] = readCoordinate(di);
}
return coordinates;
}
private Coordinate readCoordinate(DataInput di) throws IOException {
return new Coordinate(di.readFloat(), di.readFloat());
}
}

152
edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/adapters/FloatWKBWriter.java Normal file
View file

@ -0,0 +1,152 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/diclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.serialization.adapters;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import com.vividsolutions.jts.geom.Coordinate;
import com.vividsolutions.jts.geom.Geometry;
import com.vividsolutions.jts.geom.GeometryCollection;
import com.vividsolutions.jts.geom.LineString;
import com.vividsolutions.jts.geom.MultiLineString;
import com.vividsolutions.jts.geom.MultiPoint;
import com.vividsolutions.jts.geom.MultiPolygon;
import com.vividsolutions.jts.geom.Point;
import com.vividsolutions.jts.geom.Polygon;
import com.vividsolutions.jts.io.WKBConstants;
/**
* Class for serializing geometries in FloatWKB format. FloatWKB is a format
* based off of but completely incompatible with WKB. The only difference from
* WKB is that instead of writing every coordinate as a 8 byte double it is
* written as a 4 byte float. This cuts the size of objects in half while
* decreasing the precision.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 26, 2013 1954 bsteffen Initial creation
*
* </pre>
*
* @author bsteffen
* @version 1.0
*/
public class FloatWKBWriter {
public void writeGeometry(Geometry geom, OutputStream out)
throws IOException {
writeGeometry(geom, (DataOutput) new DataOutputStream(out));
}
private void writeGeometry(Geometry geom, DataOutput d)
throws IOException {
d.write(WKBConstants.wkbXDR);
if (geom instanceof Point)
writePoint((Point) geom, d);
else if (geom instanceof LineString)
writeLineString((LineString) geom, d);
else if (geom instanceof Polygon)
writePolygon((Polygon) geom, d);
else if (geom instanceof MultiPoint)
writeMultiPoint((MultiPoint) geom, d);
else if (geom instanceof MultiLineString)
writeMultiLineString((MultiLineString) geom, d);
else if (geom instanceof MultiPolygon)
writeMultiPolygon((MultiPolygon) geom, d);
else if (geom instanceof GeometryCollection)
writeGeometryCollection((GeometryCollection) geom, d);
else {
throw new IOException("Unknown Geometry type: "
+ geom.getClass().getSimpleName());
}
}
private void writePoint(Point p, DataOutput d) throws IOException {
d.writeInt(WKBConstants.wkbPoint);
writeCoordinate(p.getCoordinate(), d);
}
private void writeLineString(LineString ls, DataOutput d)
throws IOException {
d.writeInt(WKBConstants.wkbLineString);
writeCoordinates(ls.getCoordinates(), d);
}
private void writePolygon(Polygon polygon, DataOutput d) throws IOException {
d.writeInt(WKBConstants.wkbPolygon);
d.writeInt(polygon.getNumInteriorRing() + 1);
writeCoordinates(polygon.getExteriorRing().getCoordinates(), d);
for (int i = 0; i < polygon.getNumInteriorRing(); i += 1) {
writeCoordinates(polygon.getInteriorRingN(i).getCoordinates(), d);
}
}
private void writeMultiPoint(MultiPoint mp, DataOutput d)
throws IOException {
d.writeInt(WKBConstants.wkbMultiPoint);
writeMultiGeometry(mp, d);
}
private void writeMultiLineString(MultiLineString mls, DataOutput d)
throws IOException {
d.writeInt(WKBConstants.wkbMultiLineString);
writeMultiGeometry(mls, d);
}
private void writeMultiPolygon(MultiPolygon mp, DataOutput d)
throws IOException {
d.writeInt(WKBConstants.wkbMultiPolygon);
writeMultiGeometry(mp, d);
}
private void writeGeometryCollection(GeometryCollection gc, DataOutput d)
throws IOException {
d.writeInt(WKBConstants.wkbGeometryCollection);
writeMultiGeometry(gc, d);
}
private void writeMultiGeometry(GeometryCollection gc, DataOutput d)
throws IOException {
d.writeInt(gc.getNumGeometries());
for (int i = 0; i < gc.getNumGeometries(); i++) {
writeGeometry(gc.getGeometryN(i), d);
}
}
private void writeCoordinates(Coordinate[] coordinates, DataOutput d)
throws IOException {
d.writeInt(coordinates.length);
for (Coordinate c : coordinates) {
writeCoordinate(c, d);
}
}
private void writeCoordinate(Coordinate c, DataOutput d) throws IOException {
d.writeFloat((float) c.x);
d.writeFloat((float) c.y);
}
}