From 861a7b0cd5b475de9ffdd0a6c096eb2e83812a60 Mon Sep 17 00:00:00 2001 From: Richard Peter Date: Wed, 16 Apr 2014 17:12:11 -0500 Subject: [PATCH] Issue #2928: Limit thriftsrv and textdbsrv streams. Change-Id: I5f59d3ac39f7e776f88d65f3d015862968c3e178 Former-commit-id: 96bf7347434344901965bd4510d14edb9bdcd62a --- .../uf/common/serialization/JAXBManager.java | 21 ++- .../serialization/SerializationUtil.java | 22 ++- .../META-INF/MANIFEST.MF | 5 +- .../com/raytheon/uf/common/util/SizeUtil.java | 7 +- .../util/stream/CountingInputStream.java | 175 ++++++++++++++++++ .../util/stream/CountingOutputStream.java | 133 +++++++++++++ .../util/stream/LimitingInputStream.java | 81 ++++++++ .../util/stream/LimitingOutputStream.java | 81 ++++++++ .../META-INF/MANIFEST.MF | 5 +- .../build.properties | 3 +- .../res/spring/auth-common.xml | 1 + .../resources/thriftService.properties | 4 + .../edex/auth/RemoteRequestRouteWrapper.java | 57 +++++- .../.settings/org.eclipse.jdt.core.prefs | 12 +- .../src/com/raytheon/uf/edex/esb/Main.java | 2 +- .../raytheon/uf/edex/esb/camel/Executor.java | 17 +- .../META-INF/MANIFEST.MF | 3 +- .../build.properties | 3 +- .../res/spring/textdbsrv-request.xml | 9 +- .../resources/textdbsrv.properties | 4 + .../uf/edex/textdbsrv/TextDBSrvWrapper.java | 152 +++++++++++++++ 21 files changed, 761 insertions(+), 36 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingInputStream.java create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingOutputStream.java create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingInputStream.java create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingOutputStream.java create mode 100644 edexOsgi/com.raytheon.uf.edex.auth/resources/thriftService.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.textdbsrv/resources/textdbsrv.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.textdbsrv/src/com/raytheon/uf/edex/textdbsrv/TextDBSrvWrapper.java diff --git a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java index 0fb26ad000..ff2f8fcc6b 100644 --- a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java +++ b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/JAXBManager.java @@ -57,6 +57,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority; * Aug 18, 2013 #2097 dhladky Allowed extension by OGCJAXBManager * Sep 30, 2013 2361 njensen Refactored for cleanliness * Nov 14, 2013 2361 njensen Added lazy init option, improved unmarshal error message + * Apr 16, 2014 2928 rjpeter Updated marshalToStream to not close the stream. * * * @author chammack @@ -372,13 +373,22 @@ public class JAXBManager { */ public void marshalToXmlFile(Object obj, String filePath, boolean formattedOutput) throws SerializationException { + OutputStream os = null; try { - marshalToStream(obj, new FileOutputStream(new File(filePath)), - formattedOutput); + os = new FileOutputStream(new File(filePath)); + marshalToStream(obj, os, formattedOutput); } catch (SerializationException e) { throw e; } catch (Exception e) { throw new SerializationException(e); + } finally { + if (os != null) { + try { + os.close(); + } catch (IOException e) { + // ignore + } + } } } @@ -419,13 +429,6 @@ public class JAXBManager { if ((msh != null) && (marshallers.size() < QUEUE_SIZE)) { marshallers.add(msh); } - if (out != null) { - try { - out.close(); - } catch (IOException e) { - // ignore - } - } } } diff --git a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/SerializationUtil.java b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/SerializationUtil.java index 0dd8df7119..e0bf583d53 100644 --- a/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/SerializationUtil.java +++ b/edexOsgi/com.raytheon.uf.common.serialization/src/com/raytheon/uf/common/serialization/SerializationUtil.java @@ -50,7 +50,7 @@ import com.raytheon.uf.common.util.ServiceLoaderUtil; * Aug 06, 2013 2228 njensen More efficient transformFromThrift(Class, byte[]) * Aug 13, 2013 2169 bkowal Unzip any gzipped data before applying thrift transformations * Oct 01, 2013 2163 njensen Updated calls to JAXBManager - * + * Apr 16, 2014 2928 rjpeter Added jaxbMarshalToStream. * * * @author chammack @@ -176,6 +176,26 @@ public final class SerializationUtil { } + /** + * Convert an instance of a class to an XML representation and write XML to + * a stream. Uses JAXB. + * + * @param obj + * Object to be marshaled + * @param filePath + * Path to the output file + * @throws SerializationException + */ + public static void jaxbMarshalToStream(Object obj, OutputStream os) + throws SerializationException { + try { + getJaxbManager().marshalToStream(obj, os); + } catch (JAXBException e) { + throw new SerializationException(e); + } + + } + /** * Instantiates an object from the XML representation in a File. Uses JAXB. * diff --git a/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF index 76af75665b..d82760aedd 100644 --- a/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Utility Plug-in Bundle-SymbolicName: com.raytheon.uf.common.util -Bundle-Version: 1.12.1174.qualifier +Bundle-Version: 1.14.0 Bundle-Vendor: RAYTHEON Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Require-Bundle: org.apache.commons.beanutils;bundle-version="1.8.3", @@ -18,4 +18,5 @@ Export-Package: com.raytheon.uf.common.util, com.raytheon.uf.common.util.header, com.raytheon.uf.common.util.mapping, com.raytheon.uf.common.util.registry, - com.raytheon.uf.common.util.session + com.raytheon.uf.common.util.session, + com.raytheon.uf.common.util.stream diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/SizeUtil.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/SizeUtil.java index e5ee17286a..76d84fc07d 100644 --- a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/SizeUtil.java +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/SizeUtil.java @@ -39,12 +39,15 @@ package com.raytheon.uf.common.util; */ public class SizeUtil { - private static final int BYTES_PER = 1024; private static final String[] REP_PREFIX = new String[] { "B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" }; + public static final long BYTES_PER_KB = BYTES_PER; + + public static final long BYTES_PER_MB = BYTES_PER_KB * BYTES_PER; + /** * Transforms a number of bytes to a pretty string based on the total number * of bytes, e.g. B, kB, MB, or GB as fitting. For example: 1000 -> 1000B, @@ -65,7 +68,7 @@ public class SizeUtil { } int reps = 0; - while (n > BYTES_PER && reps < REP_PREFIX.length - 1) { + while ((n > BYTES_PER) && (reps < (REP_PREFIX.length - 1))) { reps++; n /= BYTES_PER; } diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingInputStream.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingInputStream.java new file mode 100644 index 0000000000..102cf06fcb --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingInputStream.java @@ -0,0 +1,175 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.stream; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Stream that counts the number of bytes that have been read. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 15, 2014 2928       rjpeter     Initial creation
+ * 
+ *
+ * + * @author rjpeter + * @version 1.0 + */ +public class CountingInputStream extends InputStream { + /** + * Stream to get data from. + */ + protected final InputStream wrappedStream; + + /** + * Number of bytes that have been read. + */ + protected long bytesRead = 0; + + /** + * Wraps the passed {@code InputStream} counting the bytes that are read + * from it. + * + * @param inputStream + */ + public CountingInputStream(InputStream inputStream) { + this.wrappedStream = inputStream; + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read(byte[]) + */ + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read(byte[], int, int) + */ + @Override + public int read(byte[] b, int off, int len) throws IOException { + int rval = wrappedStream.read(b, off, len); + increaseBytesRead(rval); + return rval; + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#skip(long) + */ + @Override + public long skip(long n) throws IOException { + return wrappedStream.skip(n); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#available() + */ + @Override + public int available() throws IOException { + return wrappedStream.available(); + } + + /** + * Closes the underlying stream. Nothing in this stream needs to be closed + * as long as the wrappedStream is closed. + */ + @Override + public void close() throws IOException { + wrappedStream.close(); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#mark(int) + */ + @Override + public synchronized void mark(int readlimit) { + wrappedStream.mark(readlimit); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#reset() + */ + @Override + public synchronized void reset() throws IOException { + wrappedStream.reset(); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#markSupported() + */ + @Override + public boolean markSupported() { + return wrappedStream.markSupported(); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read() + */ + @Override + public int read() throws IOException { + int rval = wrappedStream.read(); + increaseBytesRead(1); + return rval; + } + + /** + * Method that updates the internal count of the number of bytes read. Also + * useful extension point for special handling based on amount of bytes + * read. + * + * @param bytesRead + * @throws IOException + */ + public void increaseBytesRead(int bytesRead) throws IOException { + this.bytesRead += bytesRead; + } + + /** + * Returns the bytes read so far. + * + * @return + */ + public long getBytesRead() { + return bytesRead; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingOutputStream.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingOutputStream.java new file mode 100644 index 0000000000..e7e6a69320 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/CountingOutputStream.java @@ -0,0 +1,133 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.stream; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Stream that counts the number of bytes that have been written. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 15, 2014 2928       rjpeter     Initial creation
+ * 
+ *
+ * + * @author rjpeter + * @version 1.0 + */ +public class CountingOutputStream extends OutputStream { + /** + * Stream to write data to. + */ + protected final OutputStream wrappedStream; + + /** + * Number of bytes that have been written. + */ + protected long bytesWritten = 0; + + /** + * Wraps the passed {@code OutputStream} counting the bytes that are written + * to it. + * + * @param outputStream + */ + public CountingOutputStream(OutputStream outputStream) { + this.wrappedStream = outputStream; + } + + /* + * (non-Javadoc) + * + * @see java.io.OutputStream#write(byte[]) + */ + @Override + public void write(byte[] b) throws IOException { + this.write(b, 0, b.length); + } + + /* + * (non-Javadoc) + * + * @see java.io.OutputStream#write(byte[], int, int) + */ + @Override + public void write(byte[] b, int off, int len) throws IOException { + wrappedStream.write(b, off, len); + increaseBytesWritten(len); + } + + /* + * (non-Javadoc) + * + * @see java.io.OutputStream#flush() + */ + @Override + public void flush() throws IOException { + wrappedStream.flush(); + } + + /** + * Closes the underlying stream. Nothing in this stream needs to be closed + * as long as the wrappedStream is closed. + */ + @Override + public void close() throws IOException { + wrappedStream.close(); + } + + /* + * (non-Javadoc) + * + * @see java.io.OutputStream#write(int) + */ + @Override + public void write(int b) throws IOException { + wrappedStream.write(b); + increaseBytesWritten(1); + } + + /** + * Method that updates the internal count of the number of bytes written. + * Also useful extension point for special handling based on amount of bytes + * written. + * + * @param bytesRead + * @throws IOException + */ + public void increaseBytesWritten(int bytesWritten) throws IOException { + this.bytesWritten += bytesWritten; + } + + /** + * Returns the bytes written so far. + * + * @return + */ + public long getBytesWritten() { + return bytesWritten; + } +} diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingInputStream.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingInputStream.java new file mode 100644 index 0000000000..7d54a9c80b --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingInputStream.java @@ -0,0 +1,81 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.stream; + +import java.io.IOException; +import java.io.InputStream; + +import com.raytheon.uf.common.util.SizeUtil; + +/** + * Stream that limits the number of bytes that can be read. If limit is reached + * an IOException is thrown. This does not preclude more bytes being read from + * the stream. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 15, 2014 2928       rjpeter     Initial creation
+ * 
+ *
+ * + * @author rjpeter + * @version 1.0 + */ +public class LimitingInputStream extends CountingInputStream { + /** + * Maximum number of bytes that can be read from the wrapped stream before + * errors are thrown on read. + */ + protected final long maxBytes; + + /** + * Wraps the given {@code InputStream} and will throw IOException once the + * specified number of bytes have been read from the stream. + * + * @param inputStream + * @param maxBytes + */ + public LimitingInputStream(InputStream inputStream, long maxBytes) { + super(inputStream); + this.maxBytes = maxBytes; + } + + /** + * Tracks number of bytes read from wrapped stream. An IOException will be + * thrown if number of bytes read exceeds {@code maxBytes}. + * + * @param bytesRead + * @throws IOException + */ + @Override + public void increaseBytesRead(int bytesRead) throws IOException { + super.increaseBytesRead(bytesRead); + long curBytes = getBytesRead(); + if (curBytes > maxBytes) { + throw new IOException("Maximum number of bytes [" + + SizeUtil.prettyByteSize(maxBytes) + + "] has been exceeded by stream"); + } + } +} diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingOutputStream.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingOutputStream.java new file mode 100644 index 0000000000..68a29d84e3 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/stream/LimitingOutputStream.java @@ -0,0 +1,81 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.stream; + +import java.io.IOException; +import java.io.OutputStream; + +import com.raytheon.uf.common.util.SizeUtil; + +/** + * Stream that limits the number of bytes that can be written. If limit is + * reached an IOException is thrown. This does not preclude more bytes from + * being written to the stream. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 15, 2014 2928       rjpeter     Initial creation
+ * 
+ *
+ * + * @author rjpeter + * @version 1.0 + */ +public class LimitingOutputStream extends CountingOutputStream { + /** + * Maximum number of bytes that can be written to the wrapped stream before + * errors are thrown on write. + */ + protected final long maxBytes; + + /** + * Wraps the given {@code OutputStream} and will throw IOException once the + * specified number of bytes have been written to the stream. + * + * @param inputStream + * @param maxBytes + */ + public LimitingOutputStream(OutputStream outputStream, long maxBytes) { + super(outputStream); + this.maxBytes = maxBytes; + } + + /** + * Tracks number of bytes written to wrapped stream. An IOException will be + * thrown if number of bytes written exceeds {@code maxBytes}. + * + * @param bytesWritten + * @throws IOException + */ + @Override + public void increaseBytesWritten(int bytesWritten) throws IOException { + super.increaseBytesWritten(bytesWritten); + long curBytes = getBytesWritten(); + if (curBytes > maxBytes) { + throw new IOException("Maximum number of bytes [" + + SizeUtil.prettyByteSize(maxBytes) + + "] has been exceeded by stream"); + } + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.auth/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.auth/META-INF/MANIFEST.MF index f3cf512e01..d69c756259 100644 --- a/edexOsgi/com.raytheon.uf.edex.auth/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.auth/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: Auth Plug-in Bundle-SymbolicName: com.raytheon.uf.edex.auth -Bundle-Version: 1.12.1174.qualifier +Bundle-Version: 1.14.0 Bundle-Vendor: RAYTHEON Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.11.31", @@ -10,7 +10,8 @@ Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.11.31", com.raytheon.uf.common.auth;bundle-version="1.0.0", com.raytheon.edex.common;bundle-version="1.11.31", com.raytheon.uf.common.status;bundle-version="1.11.31", - com.raytheon.uf.common.comm;bundle-version="1.12.1174" + com.raytheon.uf.common.comm;bundle-version="1.12.1174", + com.raytheon.uf.common.util Export-Package: com.raytheon.uf.edex.auth, com.raytheon.uf.edex.auth.authentication, com.raytheon.uf.edex.auth.req, diff --git a/edexOsgi/com.raytheon.uf.edex.auth/build.properties b/edexOsgi/com.raytheon.uf.edex.auth/build.properties index 5791d48d5f..ad8d78badb 100644 --- a/edexOsgi/com.raytheon.uf.edex.auth/build.properties +++ b/edexOsgi/com.raytheon.uf.edex.auth/build.properties @@ -2,4 +2,5 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ .,\ - res/ + res/,\ + resources/ diff --git a/edexOsgi/com.raytheon.uf.edex.auth/res/spring/auth-common.xml b/edexOsgi/com.raytheon.uf.edex.auth/res/spring/auth-common.xml index 4edb120166..5a3cc392ae 100644 --- a/edexOsgi/com.raytheon.uf.edex.auth/res/spring/auth-common.xml +++ b/edexOsgi/com.raytheon.uf.edex.auth/res/spring/auth-common.xml @@ -9,6 +9,7 @@ + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.auth/resources/thriftService.properties b/edexOsgi/com.raytheon.uf.edex.auth/resources/thriftService.properties new file mode 100644 index 0000000000..f60a52b7d8 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.auth/resources/thriftService.properties @@ -0,0 +1,4 @@ +# byte limit for input and output streams to the thrift service. +# After specified number of bytes, the stream is rejected to +# prevent jvm OutOfMemory as the client is doing something wrong. +thriftService.byteLimitInMB=100 \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.auth/src/com/raytheon/uf/edex/auth/RemoteRequestRouteWrapper.java b/edexOsgi/com.raytheon.uf.edex.auth/src/com/raytheon/uf/edex/auth/RemoteRequestRouteWrapper.java index 8b9ef16e7d..660e331284 100644 --- a/edexOsgi/com.raytheon.uf.edex.auth/src/com/raytheon/uf/edex/auth/RemoteRequestRouteWrapper.java +++ b/edexOsgi/com.raytheon.uf.edex.auth/src/com/raytheon/uf/edex/auth/RemoteRequestRouteWrapper.java @@ -19,7 +19,10 @@ **/ package com.raytheon.uf.edex.auth; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import com.raytheon.uf.common.auth.AuthException; import com.raytheon.uf.common.auth.resp.AuthServerErrorResponse; @@ -31,7 +34,10 @@ import com.raytheon.uf.common.serialization.comm.RequestWrapper; import com.raytheon.uf.common.serialization.comm.response.ServerErrorResponse; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.util.ByteArrayOutputStreamPool; import com.raytheon.uf.common.util.SizeUtil; +import com.raytheon.uf.common.util.stream.LimitingInputStream; +import com.raytheon.uf.common.util.stream.LimitingOutputStream; /** * Wrapper for camel route so Serialization exceptions can be caught and @@ -48,6 +54,7 @@ import com.raytheon.uf.common.util.SizeUtil; * reduce garbage objected generated by * camel doing the auto conversion. * Feb 06, 2014 2672 bsteffen Return error when Stream is not consumed. + * Apr 15, 2014 2928 rjpeter Limit data streams read in and written out. * * * @author mschenke @@ -55,20 +62,31 @@ import com.raytheon.uf.common.util.SizeUtil; */ public class RemoteRequestRouteWrapper { - private static final IUFStatusHandler thriftSrvLogger = UFStatus .getNamedHandler("ThriftSrvRequestLogger"); private RemoteRequestServer server; + private int byteLimitInMB; + public byte[] executeThrift(InputStream data) { + /* + * This stream does not need to be closed, Camel will handle closing of + * data + */ + InputStream inputStream = null; + OutputStream outStream = null; + try { long startTime = System.currentTimeMillis(); + inputStream = new LimitingInputStream(data, byteLimitInMB + * SizeUtil.BYTES_PER_MB); Object obj = SerializationUtil.transformFromThrift(Object.class, - data); - int remaining = data.available(); + inputStream); + + int remaining = inputStream.available(); if (remaining == 1) { - int tail = data.read(); + int tail = inputStream.read(); /* * When http proxies are being used there is a single null * character at the end of the message, no need to panic. @@ -89,7 +107,14 @@ public class RemoteRequestRouteWrapper { request = (IServerRequest) obj; } Object rvalObj = server.handleThriftRequest(request); - byte[] rval = SerializationUtil.transformToThrift(rvalObj); + + ByteArrayOutputStream baos = ByteArrayOutputStreamPool + .getInstance().getStream(); + outStream = new LimitingOutputStream(baos, byteLimitInMB + * SizeUtil.BYTES_PER_MB); + + SerializationUtil.transformToThriftUsingStream(rvalObj, outStream); + byte[] rval = baos.toByteArray(); long endTime = System.currentTimeMillis(); StringBuilder sb = new StringBuilder(300); sb.append("Handled ").append(obj.toString()).append(" in ") @@ -119,6 +144,21 @@ public class RemoteRequestRouteWrapper { "Failed to serialize throwable to client", e); return new byte[] {}; } + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + // ignore + } + } + if (outStream != null) { + try { + outStream.close(); + } catch (IOException e) { + // ignore + } + } } } @@ -130,4 +170,11 @@ public class RemoteRequestRouteWrapper { this.server = server; } + public int getByteLimitInMB() { + return byteLimitInMB; + } + + public void setByteLimitInMB(int byteLimitInMB) { + this.byteLimitInMB = byteLimitInMB; + } } diff --git a/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/.settings/org.eclipse.jdt.core.prefs b/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/.settings/org.eclipse.jdt.core.prefs index 53584022bd..7341ab1683 100644 --- a/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/.settings/org.eclipse.jdt.core.prefs +++ b/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/.settings/org.eclipse.jdt.core.prefs @@ -1,7 +1,11 @@ -#Thu Mar 26 10:32:57 CDT 2009 eclipse.preferences.version=1 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 -org.eclipse.jdt.core.compiler.compliance=1.6 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.debug.lineNumber=generate +org.eclipse.jdt.core.compiler.debug.localVariable=generate +org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/src/com/raytheon/uf/edex/esb/Main.java b/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/src/com/raytheon/uf/edex/esb/Main.java index 02fbcd6644..03208ee50e 100644 --- a/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/src/com/raytheon/uf/edex/esb/Main.java +++ b/edexOsgi/com.raytheon.uf.edex.esb.camel.launcher/src/com/raytheon/uf/edex/esb/Main.java @@ -161,7 +161,7 @@ public class Main { m.invoke(null, new Object[0]); } System.exit(0); - } catch (ClassNotFoundException e) { + } catch (ClassNotFoundException | LinkageError e) { logger.error("Could not load class", e); if (cl != null) { StringBuilder msg = new StringBuilder(1000); diff --git a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/Executor.java b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/Executor.java index 4b6fb4a58a..1931ab7d5b 100644 --- a/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/Executor.java +++ b/edexOsgi/com.raytheon.uf.edex.esb.camel/src/com/raytheon/uf/edex/esb/camel/Executor.java @@ -28,6 +28,7 @@ import java.util.Iterator; import java.util.List; import java.util.Properties; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,6 +76,10 @@ public class Executor { public static void start() throws Exception { final long t0 = System.currentTimeMillis(); + Thread.currentThread().setName("EDEXMain"); + System.setProperty("System.status", "Starting"); + final AtomicBoolean shutdownContexts = new AtomicBoolean(false); + Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { @@ -87,7 +92,12 @@ public class Executor { .append("\n* EDEX ESB is shutting down *") .append("\n**************************************************"); logger.info(msg.toString()); - ctxMgr.stopContexts(); + if (shutdownContexts.get()) { + ctxMgr.stopContexts(); + } else { + logger.info("Contexts never started, skipping context shutdown"); + } + long t2 = System.currentTimeMillis(); msg.setLength(0); msg.append("\n**************************************************"); @@ -102,9 +112,6 @@ public class Executor { } }); - Thread.currentThread().setName("EDEXMain"); - System.setProperty("System.status", "Starting"); - List xmlFiles = new ArrayList(); List propertiesFiles = new ArrayList(); @@ -166,6 +173,8 @@ public class Executor { ContextManager ctxMgr = (ContextManager) context .getBean("contextManager"); + shutdownContexts.set(true); + // start final routes ctxMgr.startContexts(); diff --git a/edexOsgi/com.raytheon.uf.edex.textdbsrv/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.textdbsrv/META-INF/MANIFEST.MF index eb903b71cf..2e269620fc 100644 --- a/edexOsgi/com.raytheon.uf.edex.textdbsrv/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.edex.textdbsrv/META-INF/MANIFEST.MF @@ -10,7 +10,8 @@ Require-Bundle: com.raytheon.edex.common, org.apache.commons.lang, com.raytheon.uf.edex.decodertools;bundle-version="1.0.0", com.raytheon.uf.common.dataplugin.text, - com.raytheon.uf.common.site;bundle-version="1.12.1152" + com.raytheon.uf.common.site;bundle-version="1.12.1152", + com.raytheon.uf.common.status Export-Package: com.raytheon.uf.edex.services, com.raytheon.uf.edex.services.textdbimpl, com.raytheon.uf.edex.services.textdbsrv diff --git a/edexOsgi/com.raytheon.uf.edex.textdbsrv/build.properties b/edexOsgi/com.raytheon.uf.edex.textdbsrv/build.properties index 5791d48d5f..0aa3712428 100644 --- a/edexOsgi/com.raytheon.uf.edex.textdbsrv/build.properties +++ b/edexOsgi/com.raytheon.uf.edex.textdbsrv/build.properties @@ -2,4 +2,5 @@ source.. = src/ output.. = bin/ bin.includes = META-INF/,\ .,\ - res/ + res/,\ + resources/ \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.textdbsrv/res/spring/textdbsrv-request.xml b/edexOsgi/com.raytheon.uf.edex.textdbsrv/res/spring/textdbsrv-request.xml index 73917fc0e1..4608bb9051 100644 --- a/edexOsgi/com.raytheon.uf.edex.textdbsrv/res/spring/textdbsrv-request.xml +++ b/edexOsgi/com.raytheon.uf.edex.textdbsrv/res/spring/textdbsrv-request.xml @@ -3,6 +3,11 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd"> + + + + + - - - + \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.textdbsrv/resources/textdbsrv.properties b/edexOsgi/com.raytheon.uf.edex.textdbsrv/resources/textdbsrv.properties new file mode 100644 index 0000000000..7b3f8ff207 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.textdbsrv/resources/textdbsrv.properties @@ -0,0 +1,4 @@ +# byte limit for input and output streams to the textdb srv. +# After specified number of bytes, the stream is rejected to +# prevent jvm OutOfMemory as the client is doing something wrong. +textdbsrv.byteLimitInMB=10 \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.textdbsrv/src/com/raytheon/uf/edex/textdbsrv/TextDBSrvWrapper.java b/edexOsgi/com.raytheon.uf.edex.textdbsrv/src/com/raytheon/uf/edex/textdbsrv/TextDBSrvWrapper.java new file mode 100644 index 0000000000..1a1a68fd07 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.textdbsrv/src/com/raytheon/uf/edex/textdbsrv/TextDBSrvWrapper.java @@ -0,0 +1,152 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + */ +package com.raytheon.uf.edex.textdbsrv; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +import com.raytheon.uf.common.message.Message; +import com.raytheon.uf.common.serialization.SerializationUtil; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.util.ByteArrayOutputStreamPool; +import com.raytheon.uf.common.util.SizeUtil; +import com.raytheon.uf.common.util.stream.LimitingInputStream; +import com.raytheon.uf.common.util.stream.LimitingOutputStream; +import com.raytheon.uf.edex.services.TextDBSrv; +import com.raytheon.uf.edex.services.textdbimpl.CommandExecutor; + +/** + * Thin wrapper around TextDBSrv to handle marshalling/unmarshalling and + * limiting of the byte stream. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Apr 15, 2014 2928       rjpeter     Initial creation.
+ * 
+ *
+ * + * @author rjpeter + * @version 1.0 + */ +public class TextDBSrvWrapper { + private final IUFStatusHandler statusHandler = UFStatus + .getHandler(TextDBSrvWrapper.class); + + /** + * The limit of bytes that we are able to read in without erroring off. + */ + private long byteLimitInMB; + + /** + * TextDbSrv implementation to use. + */ + private TextDBSrv textdbSrv; + + /** + * Unmarshalls the input stream as xml data and sends to textdbsrv for + * processing. + * + * @param is + * @return + */ + public byte[] executeTextDBMessage(InputStream xmlDataStream) { + /* + * This stream does not need to be closed, Camel will handle closing of + * data + */ + InputStream inputStream = null; + Message rval; + + try { + inputStream = new LimitingInputStream(xmlDataStream, byteLimitInMB + * SizeUtil.BYTES_PER_MB); + Message message = SerializationUtil.jaxbUnmarshalFromInputStream( + Message.class, inputStream); + rval = textdbSrv.processMessage(message); + } catch (Throwable e) { + statusHandler + .error("Error occured processing textDbSrv message", e); + rval = CommandExecutor + .createErrorMessage("Error occurred during processing: " + + e.getLocalizedMessage()); + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + // ignore + } + } + } + + OutputStream outStream = null; + int tries = 0; + + while (tries < 2) { + try { + ByteArrayOutputStream baos = ByteArrayOutputStreamPool + .getInstance().getStream(); + outStream = new LimitingOutputStream(baos, byteLimitInMB + * SizeUtil.BYTES_PER_MB); + SerializationUtil.jaxbMarshalToStream(rval, outStream); + return baos.toByteArray(); + } catch (Exception e) { + statusHandler.error("Error occured marshalling response", e); + tries++; + rval = CommandExecutor + .createErrorMessage("Error occurred during processing: " + + e.getLocalizedMessage()); + } finally { + if (outStream != null) { + try { + outStream.close(); + } catch (IOException e) { + // ignore + } + } + } + } + + return null; + } + + public long getByteLimitInMB() { + return byteLimitInMB; + } + + public void setByteLimitInMB(long byteLimitInMB) { + this.byteLimitInMB = byteLimitInMB; + } + + public TextDBSrv getTextdbSrv() { + return textdbSrv; + } + + public void setTextdbSrv(TextDBSrv textdbSrv) { + this.textdbSrv = textdbSrv; + } +}