xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #443 improve zlib compression

Browse source 
Change-Id: If1ce4e4288ad45276d11a4d9a3bf5dc8435352e4

Former-commit-id: 4dbc6f7726 [formerly 105d703fbf] [formerly 4dbc6f7726 [formerly 105d703fbf] [formerly 1838de6993 [formerly 5203a659ec0aaea45cbf281544b22b23b54a51d8]]]
Former-commit-id: 1838de6993
Former-commit-id: 0095b4083e [formerly 571eca8d57]
Former-commit-id: ef97a25505
This commit is contained in:
Nate Jensen 2012-05-11 14:04:41 -05:00
parent 99130351c6
commit 824d66ad32
5 changed files with 191 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cave/com.raytheon.uf.viz.collaboration.comm/META-INF/MANIFEST.MF
View file

@ -19,6 +19,7 @@ Require-Bundle: org.eclipse.core.runtime,
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Bundle-ActivationPolicy: lazy
Export-Package: com.raytheon.uf.viz.collaboration.comm,
com.raytheon.uf.viz.collaboration.comm.compression,
com.raytheon.uf.viz.collaboration.comm.identity,
com.raytheon.uf.viz.collaboration.comm.identity.event,
com.raytheon.uf.viz.collaboration.comm.identity.info,

175
cave/com.raytheon.uf.viz.collaboration.comm/src/com/raytheon/uf/viz/collaboration/comm/compression/CompressionUtil.java Normal file
View file

@ -0,0 +1,175 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.viz.collaboration.comm.compression;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.InflaterInputStream;
import com.raytheon.uf.viz.collaboration.comm.identity.CollaborationException;
/**
* Utilities for compressing or decompressing data.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* May 11, 2012 njensen Initial creation
*
* </pre>
*
* @author njensen
* @version 1.0
*/
public class CompressionUtil {
public static CompressionType COMPRESSION_TYPE = CompressionType.ZLIB;
private static boolean log_compression = false;
private enum CompressionType {
ZLIB, GZIP;
public byte toByte() {
return (byte) ordinal();
}
public static CompressionType fromByte(byte b) {
if (b < 0 || b > CompressionType.values().length) {
throw new IndexOutOfBoundsException(
"Unable to determine CompressionType for " + b);
}
return CompressionType.values()[b];
}
};
public static byte[] compress(byte[] bytes) throws CollaborationException {
ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length);
CompressionType cType = COMPRESSION_TYPE;
out.write(cType.toByte());
OutputStream compressionStrm = null;
try {
compressionStrm = createCompressionOutputStream(out);
long start = System.currentTimeMillis();
compressionStrm.write(bytes);
compressionStrm.flush();
compressionStrm.close();
byte[] result = out.toByteArray();
if (log_compression) {
System.out.println(cType + " Compression time(milliseconds) "
+ (System.currentTimeMillis() - start) / 1000F
+ " to compress " + bytes.length + " bytes to "
+ result.length + " bytes.");
}
return result;
} catch (IOException e) {
throw new CollaborationException("Unable to compress data.", e);
}
}
private static OutputStream createCompressionOutputStream(OutputStream out)
throws IOException {
OutputStream stream = null;
switch (COMPRESSION_TYPE) {
case GZIP:
stream = new GZIPOutputStream(out);
break;
case ZLIB:
default:
Deflater defl = new Deflater(Deflater.BEST_COMPRESSION);
stream = new DeflaterOutputStream(out, defl);
break;
}
return stream;
}
public static byte[] uncompress(byte[] bytes) throws CollaborationException {
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
ByteArrayOutputStream out = new ByteArrayOutputStream();
CompressionType cType = CompressionType.fromByte((byte) in.read());
long start = System.currentTimeMillis();
try {
ReadableByteChannel src = Channels
.newChannel(createCompressionInputStream(cType, in));
WritableByteChannel dest = Channels.newChannel(out);
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
dest.write(buffer);
buffer.compact();
}
// EOF will leave buffer in fill state
buffer.flip();
// make sure the buffer is fully drained.
while (buffer.hasRemaining()) {
dest.write(buffer);
}
dest.close();
byte[] resultBuffer = out.toByteArray();
if (log_compression) {
System.out.println(cType
+ " Uncompression time(milliseconds): "
+ ((System.currentTimeMillis() - start) / 1000F)
+ " to uncompress " + bytes.length + " bytes to "
+ resultBuffer.length + " bytes.");
}
return resultBuffer;
} catch (IOException e) {
throw new CollaborationException("Unable to uncompress data.", e);
}
}
private static InputStream createCompressionInputStream(
CompressionType cType, InputStream in) throws IOException {
InputStream stream = null;
switch (cType) {
case GZIP:
stream = new GZIPInputStream(in);
break;
case ZLIB:
default:
stream = new InflaterInputStream(in);
break;
}
return stream;
}
}

148
cave/com.raytheon.uf.viz.collaboration.comm/src/com/raytheon/uf/viz/collaboration/comm/provider/Tools.java
View file

@ -19,20 +19,6 @@
**/
package com.raytheon.uf.viz.collaboration.comm.provider;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import java.util.zip.InflaterInputStream;
import javax.xml.bind.JAXBException;
import org.eclipse.ecf.core.IContainer;
@ -40,6 +26,7 @@ import org.eclipse.ecf.core.util.Base64;
import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.viz.collaboration.comm.compression.CompressionUtil;
import com.raytheon.uf.viz.collaboration.comm.identity.CollaborationException;
/**
@ -100,35 +87,10 @@ public abstract class Tools {
public static boolean COMPRESSION_OFF = false;
public static CompressionType COMPRESSION_TYPE = CompressionType.ZLIB;
private static boolean log_compression = false;
private enum CompressionType {
ZLIB, GZIP;
public byte toByte() {
return (byte) ordinal();
}
public static CompressionType fromByte(byte b) {
if (b < 0 || b > CompressionType.values().length) {
throw new IndexOutOfBoundsException(
"Unable to determine CompressionType for " + b);
}
return CompressionType.values()[b];
}
};
static {
try {
COMPRESSION_OFF = Boolean
.getBoolean("collaboration.compressionOff");
String compressionType = System
.getProperty("collaboration.compressionType");
if (compressionType != null) {
COMPRESSION_TYPE = CompressionType.valueOf(compressionType);
}
} catch (Exception e) {
// must not have permission to access system properties. ignore and
// use default.
@ -297,7 +259,8 @@ public abstract class Tools {
*/
byte[] marshalledThrift = SerializationUtil
.transformToThrift(data);
marshalledBinary = compress(marshalledThrift);
marshalledBinary = CompressionUtil
.compress(marshalledThrift);
sb.append(ENV_THRIFT_COMPRESSED);
}
} catch (Exception e) {
@ -316,7 +279,8 @@ public abstract class Tools {
}
} else {
String rawString = SerializationUtil.marshalToXml(data);
marshalledBinary = compress(rawString.getBytes());
marshalledBinary = CompressionUtil.compress(rawString
.getBytes());
sb.append(ENV_JAXB_COMPRESSED);
}
} catch (Exception je) {
@ -375,7 +339,8 @@ public abstract class Tools {
String s = data.substring(ENV_THRIFT_COMPRESSED.length());
try {
byte[] rawBytes = decodeFromBase64(s);
byte[] uncompressedBytes = uncompress(rawBytes);
byte[] uncompressedBytes = CompressionUtil
.uncompress(rawBytes);
unMarshalledData = SerializationUtil
.transformFromThrift(uncompressedBytes);
@ -398,7 +363,8 @@ public abstract class Tools {
try {
byte[] rawBytes = decodeFromBase64(rawString);
unMarshalledData = SerializationUtil
.unmarshalFromXml(new String(uncompress(rawBytes)));
.unmarshalFromXml(new String(CompressionUtil
.uncompress(rawBytes)));
// unMarshalledData = SerializationUtil
// .unmarshalFromXml(createCompressionInputStream(rawBytes));
} catch (Exception je) {
@ -416,100 +382,4 @@ public abstract class Tools {
return unMarshalledData;
}
public static byte[] compress(byte[] bytes) throws CollaborationException {
ByteArrayOutputStream out = new ByteArrayOutputStream(bytes.length);
CompressionType cType = COMPRESSION_TYPE;
out.write(cType.toByte());
OutputStream compressionStrm = null;
try {
compressionStrm = createCompressionOutputStream(out);
long start = System.currentTimeMillis();
compressionStrm.write(bytes);
compressionStrm.flush();
compressionStrm.close();
byte[] result = out.toByteArray();
if (log_compression) {
System.out.println(cType + " Compression time(milliseconds) "
+ (System.currentTimeMillis() - start) / 1000F
+ " to compress " + bytes.length + " bytes to "
+ result.length + " bytes.");
}
return result;
} catch (IOException e) {
throw new CollaborationException("Unable to compress data.", e);
}
}
private static OutputStream createCompressionOutputStream(OutputStream out)
throws IOException {
OutputStream stream = null;
switch (COMPRESSION_TYPE) {
case GZIP:
stream = new GZIPOutputStream(out);
break;
case ZLIB:
default:
stream = new DeflaterOutputStream(out);
break;
}
return stream;
}
public static byte[] uncompress(byte[] bytes) throws CollaborationException {
ByteArrayInputStream in = new ByteArrayInputStream(bytes);
ByteArrayOutputStream out = new ByteArrayOutputStream();
CompressionType cType = CompressionType.fromByte((byte) in.read());
long start = System.currentTimeMillis();
try {
ReadableByteChannel src = Channels
.newChannel(createCompressionInputStream(cType, in));
WritableByteChannel dest = Channels.newChannel(out);
final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024);
while (src.read(buffer) != -1) {
buffer.flip();
dest.write(buffer);
buffer.compact();
}
// EOF will leave buffer in fill state
buffer.flip();
// make sure the buffer is fully drained.
while (buffer.hasRemaining()) {
dest.write(buffer);
}
dest.close();
byte[] resultBuffer = out.toByteArray();
if (log_compression) {
System.out.println(cType
+ " Uncompression time(milliseconds): "
+ ((System.currentTimeMillis() - start) / 1000F)
+ " to uncompress " + bytes.length + " bytes to "
+ resultBuffer.length + " bytes.");
}
return resultBuffer;
} catch (IOException e) {
throw new CollaborationException("Unable to uncompress data.", e);
}
}
private static InputStream createCompressionInputStream(
CompressionType cType, InputStream in) throws IOException {
InputStream stream = null;
switch (cType) {
case GZIP:
stream = new GZIPInputStream(in);
break;
case ZLIB:
default:
stream = new InflaterInputStream(in);
break;
}
return stream;
}
}

4
cave/com.raytheon.uf.viz.collaboration.ui/src/com/raytheon/uf/viz/collaboration/ui/role/dataprovider/CollaborationDispatcher.java
View file

@ -28,9 +28,9 @@ import org.eclipse.swt.widgets.Event;
import com.google.common.eventbus.Subscribe;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.viz.collaboration.comm.compression.CompressionUtil;
import com.raytheon.uf.viz.collaboration.comm.identity.CollaborationException;
import com.raytheon.uf.viz.collaboration.comm.identity.ISharedDisplaySession;
import com.raytheon.uf.viz.collaboration.comm.provider.Tools;
import com.raytheon.uf.viz.collaboration.ui.Activator;
import com.raytheon.uf.viz.collaboration.ui.role.dataprovider.event.IPersistedEvent;
import com.raytheon.uf.viz.collaboration.ui.role.dataprovider.event.IRenderFrameEvent;
@ -176,7 +176,7 @@ public class CollaborationDispatcher extends Dispatcher {
// Not a creation event, check event size. All creation events
// are sent immediately to avoid false negatives
try {
byte[] data = Tools.compress(SerializationUtil
byte[] data = CompressionUtil.compress(SerializationUtil
.transformToThrift(eventObject));
if (data.length > IMMEDIATE_SEND_SIZE) {
immediateSend = false;

7
cave/com.raytheon.uf.viz.collaboration.ui/src/com/raytheon/uf/viz/collaboration/ui/role/dataprovider/CollaborationObjectEventStorage.java
View file

@ -39,9 +39,9 @@ import com.raytheon.uf.common.serialization.SerializationException;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.viz.collaboration.comm.compression.CompressionUtil;
import com.raytheon.uf.viz.collaboration.comm.identity.CollaborationException;
import com.raytheon.uf.viz.collaboration.comm.identity.ISharedDisplaySession;
import com.raytheon.uf.viz.collaboration.comm.provider.Tools;
import com.raytheon.uf.viz.collaboration.ui.Activator;
import com.raytheon.uf.viz.collaboration.ui.prefs.CollabPrefConstants;
import com.raytheon.uf.viz.collaboration.ui.role.dataprovider.event.IPersistedEvent;
@ -135,7 +135,7 @@ public class CollaborationObjectEventStorage implements
CollaborationHttpPersistedObject persistObject = new CollaborationHttpPersistedObject();
persistObject.persistTime = System.currentTimeMillis();
persistObject.event = event;
byte[] toPersist = Tools.compress(SerializationUtil
byte[] toPersist = CompressionUtil.compress(SerializationUtil
.transformToThrift(persistObject));
stats.log(event.getClass().getSimpleName(), toPersist.length, 0);
put.setEntity(new ByteArrayEntity(toPersist));
@ -208,7 +208,8 @@ public class CollaborationObjectEventStorage implements
if (isSuccess(response.code)) {
try {
CollaborationHttpPersistedObject dataObject = (CollaborationHttpPersistedObject) SerializationUtil
.transformFromThrift(Tools.uncompress(response.data));
.transformFromThrift(CompressionUtil
.uncompress(response.data));
if (dataObject != null) {
dataObject.dataSize = response.data.length;
}