From 51f9666fcb55d7e7a9a01f4fc36a7c9cd2b69967 Mon Sep 17 00:00:00 2001 From: Max Schenkelberg Date: Thu, 26 Jul 2012 17:37:07 -0500 Subject: [PATCH] Issue #626 Moved data unzipper and wmo header remover to common so it can potentially be used by GOES-R Amend: Added WMO header finder since the header property will have additional information in it when file comes from LDM. Also changed MessageGenerator to use new class for finding the wmo header Amend: Added try-finally for input stream in DataUnzipper Change-Id: I4ece2f5f24a429aa41969f51040a58e67e79e1c1 Former-commit-id: 756988cb8bd86000c7dc7bdd7dd48340498fa8a1 [formerly 2f17b2f49f65da4b7b7fd531377873cb9eb941dd] Former-commit-id: b669f57ee4ef2605a15639bde9c971e6ac90a6ea --- edexOsgi/build.edex/esb/conf/spring/edex.xml | 4 +- .../META-INF/MANIFEST.MF | 1 + .../raytheon/uf/common/util/DataUnzipper.java | 94 ++++++++++++++++ .../common/util/header/WMOHeaderFinder.java | 102 ++++++++++++++++++ .../common/util/header/WMOHeaderRemover.java | 72 +++++++++++++ .../plugin/manualIngest/MessageGenerator.java | 52 +++------ 6 files changed, 284 insertions(+), 41 deletions(-) create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java create mode 100644 edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java diff --git a/edexOsgi/build.edex/esb/conf/spring/edex.xml b/edexOsgi/build.edex/esb/conf/spring/edex.xml index 094a2c6e62..ceac377b29 100644 --- a/edexOsgi/build.edex/esb/conf/spring/edex.xml +++ b/edexOsgi/build.edex/esb/conf/spring/edex.xml @@ -154,7 +154,9 @@ - + + + diff --git a/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF index 9c988037d0..d4b62e6588 100644 --- a/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF +++ b/edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF @@ -8,5 +8,6 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Require-Bundle: org.junit;bundle-version="1.0.0" Export-Package: com.raytheon.uf.common.util, com.raytheon.uf.common.util.cache, + com.raytheon.uf.common.util.header, com.raytheon.uf.common.util.registry Import-Package: org.apache.commons.lang diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java new file mode 100644 index 0000000000..7438a23784 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java @@ -0,0 +1,94 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipInputStream; + +/** + * Bean that will attempt to decompress a byte[]. Will throw IOException if data + * can not be unzipped + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 17, 2012            mschenke     Initial creation
+ * 
+ *
+ * + * @author mschenke + * @version 1.0 + */ + +public class DataUnzipper { + + /** + * Uses ZLIB decompression to unzip the data + * + * @param data + * @return + * @throws IOException + */ + public byte[] unzip(byte[] data) throws IOException { + return unzip(new ZipInputStream(new ByteArrayInputStream(data)), + data.length * 2); + } + + /** + * Uses GZIP decompression to unzip the data + * + * @param data + * @return + * @throws IOException + */ + public byte[] gunzip(byte[] data) throws IOException { + return unzip(new GZIPInputStream(new ByteArrayInputStream(data)), + data.length * 2); + } + + /** + * Unzips the {@link InputStream} into a byte[] this function will close the + * in stream after decompression + * + * @param in + * @param size + * @return + * @throws IOException + */ + private byte[] unzip(InputStream in, int size) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(size); + try { + byte[] buf = new byte[4096]; + int len; + while ((len = in.read(buf)) > 0) + out.write(buf, 0, len); + } finally { + in.close(); + } + return out.toByteArray(); + } +} diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java new file mode 100644 index 0000000000..4e29123be7 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java @@ -0,0 +1,102 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.header; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utility class for finding a WMO header + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 27, 2012            mschenke     Initial creation
+ * 
+ *
+ * + * @author mschenke + * @version 1.0 + */ + +public class WMOHeaderFinder { + + private static Pattern WMOPATTERN = Pattern + .compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} " + + "\\d{6}[^\\r\\n]*)[\\r\\n]+"); + + /** + * Finds and returns the WMO header on the {@link File} + * + * @param file + * @return + * @throws FileNotFoundException + * @throws IOException + */ + public static String find(File file) throws FileNotFoundException, + IOException { + return find(new FileInputStream(file)); + } + + /** + * Finds and returns the WMO header on the data + * + * @param data + * @return + * @throws IOException + */ + public static String find(byte[] data) throws IOException { + return find(new ByteArrayInputStream(data)); + } + + /** + * Finds and returns the WMO header from the {@link InputStream}. Didn't + * expose this function because I wanted to ensure that it was responsible + * for closing the input stream + * + * @param in + * @return + * @throws IOException + */ + private static String find(InputStream in) throws IOException { + try { + byte[] header = new byte[100]; + in.read(header); + String sHeader = new String(header, "ISO-8859-1"); + Matcher wmoSearch = WMOPATTERN.matcher(sHeader); + String messageHeader = null; + if (wmoSearch.find()) { + messageHeader = wmoSearch.group(); + } + return messageHeader; + } finally { + in.close(); + } + } +} diff --git a/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java new file mode 100644 index 0000000000..82f2287046 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java @@ -0,0 +1,72 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.common.util.header; + +import java.io.IOException; + +/** + * Bean that will attempt to extract the WMO header found in the {@link Headers} + * argument from the begining of the byte[] + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 17, 2012            mschenke     Initial creation
+ * 
+ *
+ * + * @author mschenke + * @version 1.0 + */ + +public class WMOHeaderRemover { + + public byte[] remove(byte[] data) throws IOException { + String header = WMOHeaderFinder.find(data); + if (header == null) { + // No header found, nothing to do + return data; + } + + byte[] headerBytes = header.getBytes(); + + boolean equal = false; + // Do < since there should be something after the header + if (headerBytes.length < data.length) { + equal = true; + for (int i = 0; i < headerBytes.length; ++i) { + if (headerBytes[i] != data[i]) { + equal = false; + break; + } + } + } + if (equal) { + byte[] newData = new byte[data.length - headerBytes.length]; + System.arraycopy(data, headerBytes.length, newData, 0, + newData.length); + data = newData; + } + return data; + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java b/edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java index cb05e7b8d8..4609c77b99 100644 --- a/edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java +++ b/edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java @@ -20,11 +20,7 @@ package com.raytheon.uf.edex.plugin.manualIngest; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.camel.Exchange; import org.apache.camel.Processor; @@ -33,6 +29,7 @@ import org.springframework.util.FileCopyUtils; import com.raytheon.uf.common.status.IUFStatusHandler; import com.raytheon.uf.common.status.UFStatus; import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.common.util.header.WMOHeaderFinder; import com.raytheon.uf.edex.core.EDEXUtil; import com.raytheon.uf.edex.core.EdexException; import com.raytheon.uf.edex.core.props.PropertiesFactory; @@ -64,10 +61,6 @@ public class MessageGenerator implements Processor { .getEnvProperties().getEnvValue("ARCHIVEDIR") + File.separator + "manual"; - private static Pattern WMOPATTERN = Pattern - .compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} " - + "\\d{6}[^\\r\\n]*)[\\r\\n]+"); - private static MessageGenerator instance = new MessageGenerator(); private String ingestRoute = null; @@ -91,39 +84,17 @@ public class MessageGenerator implements Processor { */ @Override public void process(Exchange arg0) throws Exception { - - byte[] header = new byte[100]; - File file = (File) arg0.getIn().getBody(); - if (file != null) { String fileName = file.getName(); - - InputStream is = null; - try { - is = new FileInputStream(file); - is.read(header); - String sHeader = new String(header, "ISO-8859-1"); - Matcher wmoSearch = WMOPATTERN.matcher(sHeader); - String messageHeader = fileName; - - if (wmoSearch.find()) { - messageHeader = wmoSearch.group(); - } - - arg0.getIn().setBody(file.toString()); - arg0.getIn().setHeader("header", messageHeader); - arg0.getIn().setHeader("enqueueTime", - System.currentTimeMillis()); - } finally { - if (is != null) { - try { - is.close(); - } catch (IOException e) { - // ignore - } - } + String messageHeader = WMOHeaderFinder.find(file); + if (messageHeader == null) { + messageHeader = fileName; } + + arg0.getIn().setBody(file.toString()); + arg0.getIn().setHeader("header", messageHeader); + arg0.getIn().setHeader("enqueueTime", System.currentTimeMillis()); } else { // No file received arg0.getOut().setFault(true); @@ -135,7 +106,7 @@ public class MessageGenerator implements Processor { // Determine the sub-directory String inputPath = inFile.getParent(); - + // Split on the manual directory to get the sub-directory String[] parts = inputPath.split("manual"); File dir = null; @@ -144,7 +115,7 @@ public class MessageGenerator implements Processor { } else { dir = new File(path); } - + if (!dir.exists()) { dir.mkdirs(); } @@ -153,7 +124,8 @@ public class MessageGenerator implements Processor { try { FileCopyUtils.copy(inFile, newFile); - statusHandler.handle(Priority.INFO, "DataManual: " + inFile.getAbsolutePath()); + statusHandler.handle(Priority.INFO, + "DataManual: " + inFile.getAbsolutePath()); } catch (IOException e) { statusHandler.handle(Priority.ERROR, "Failed to copy file [" + inFile.getAbsolutePath() + "] to archive dir", e);