Merge "Issue #626 Moved data unzipper and wmo header remover to common so it can potentially be used by GOES-R" into development

Former-commit-id: 9e3cd2546a [formerly 39e74c7e2aa9f78938c85d0433decb89f4ef1325]
Former-commit-id: 77c2ab2c05

edexOsgi/build.edex/esb/conf/spring/edex.xml

@@ -154,7 +154,9 @@
     	<constructor-arg><ref bean="pluginSetup"/></constructor-arg>
     </bean>
     
-    <bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>       
+    <bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>
+    <bean id="extractWMOHeader" class="com.raytheon.uf.common.util.header.WMOHeaderRemover"/>
+	<bean id="dataUnzipper" class="com.raytheon.uf.common.util.DataUnzipper"/>       
     
     <bean id="errorHandler" class="org.apache.camel.builder.LoggingErrorHandlerBuilder"/>
 

edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF

@@ -8,5 +8,6 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
 Require-Bundle: org.junit;bundle-version="1.0.0"
 Export-Package: com.raytheon.uf.common.util,
  com.raytheon.uf.common.util.cache,
+ com.raytheon.uf.common.util.header,
  com.raytheon.uf.common.util.registry
 Import-Package: org.apache.commons.lang

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java

@@ -0,0 +1,94 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipInputStream;
+
+/**
+ * Bean that will attempt to decompress a byte[]. Will throw IOException if data
+ * can not be unzipped
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 17, 2012            mschenke     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author mschenke
+ * @version 1.0
+ */
+
+public class DataUnzipper {
+
+    /**
+     * Uses ZLIB decompression to unzip the data
+     * 
+     * @param data
+     * @return
+     * @throws IOException
+     */
+    public byte[] unzip(byte[] data) throws IOException {
+        return unzip(new ZipInputStream(new ByteArrayInputStream(data)),
+                data.length * 2);
+    }
+
+    /**
+     * Uses GZIP decompression to unzip the data
+     * 
+     * @param data
+     * @return
+     * @throws IOException
+     */
+    public byte[] gunzip(byte[] data) throws IOException {
+        return unzip(new GZIPInputStream(new ByteArrayInputStream(data)),
+                data.length * 2);
+    }
+
+    /**
+     * Unzips the {@link InputStream} into a byte[] this function will close the
+     * in stream after decompression
+     * 
+     * @param in
+     * @param size
+     * @return
+     * @throws IOException
+     */
+    private byte[] unzip(InputStream in, int size) throws IOException {
+        ByteArrayOutputStream out = new ByteArrayOutputStream(size);
+        try {
+            byte[] buf = new byte[4096];
+            int len;
+            while ((len = in.read(buf)) > 0)
+                out.write(buf, 0, len);
+        } finally {
+            in.close();
+        }
+        return out.toByteArray();
+    }
+}

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java

@@ -0,0 +1,102 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.util.header;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Utility class for finding a WMO header
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 27, 2012            mschenke     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author mschenke
+ * @version 1.0
+ */
+
+public class WMOHeaderFinder {
+
+    private static Pattern WMOPATTERN = Pattern
+            .compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
+                    + "\\d{6}[^\\r\\n]*)[\\r\\n]+");
+
+    /**
+     * Finds and returns the WMO header on the {@link File}
+     * 
+     * @param file
+     * @return
+     * @throws FileNotFoundException
+     * @throws IOException
+     */
+    public static String find(File file) throws FileNotFoundException,
+            IOException {
+        return find(new FileInputStream(file));
+    }
+
+    /**
+     * Finds and returns the WMO header on the data
+     * 
+     * @param data
+     * @return
+     * @throws IOException
+     */
+    public static String find(byte[] data) throws IOException {
+        return find(new ByteArrayInputStream(data));
+    }
+
+    /**
+     * Finds and returns the WMO header from the {@link InputStream}. Didn't
+     * expose this function because I wanted to ensure that it was responsible
+     * for closing the input stream
+     * 
+     * @param in
+     * @return
+     * @throws IOException
+     */
+    private static String find(InputStream in) throws IOException {
+        try {
+            byte[] header = new byte[100];
+            in.read(header);
+            String sHeader = new String(header, "ISO-8859-1");
+            Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
+            String messageHeader = null;
+            if (wmoSearch.find()) {
+                messageHeader = wmoSearch.group();
+            }
+            return messageHeader;
+        } finally {
+            in.close();
+        }
+    }
+}

edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java

@@ -0,0 +1,72 @@
+/**
+ * This software was developed and / or modified by Raytheon Company,
+ * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
+ * 
+ * U.S. EXPORT CONTROLLED TECHNICAL DATA
+ * This software product contains export-restricted data whose
+ * export/transfer/disclosure is restricted by U.S. law. Dissemination
+ * to non-U.S. persons whether in the United States or abroad requires
+ * an export license or other authorization.
+ * 
+ * Contractor Name:        Raytheon Company
+ * Contractor Address:     6825 Pine Street, Suite 340
+ *                         Mail Stop B8
+ *                         Omaha, NE 68106
+ *                         402.291.0100
+ * 
+ * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
+ * further licensing information.
+ **/
+package com.raytheon.uf.common.util.header;
+
+import java.io.IOException;
+
+/**
+ * Bean that will attempt to extract the WMO header found in the {@link Headers}
+ * argument from the begining of the byte[]
+ * 
+ * <pre>
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Jul 17, 2012            mschenke     Initial creation
+ * 
+ * </pre>
+ * 
+ * @author mschenke
+ * @version 1.0
+ */
+
+public class WMOHeaderRemover {
+
+    public byte[] remove(byte[] data) throws IOException {
+        String header = WMOHeaderFinder.find(data);
+        if (header == null) {
+            // No header found, nothing to do
+            return data;
+        }
+
+        byte[] headerBytes = header.getBytes();
+
+        boolean equal = false;
+        // Do < since there should be something after the header
+        if (headerBytes.length < data.length) {
+            equal = true;
+            for (int i = 0; i < headerBytes.length; ++i) {
+                if (headerBytes[i] != data[i]) {
+                    equal = false;
+                    break;
+                }
+            }
+        }
+        if (equal) {
+            byte[] newData = new byte[data.length - headerBytes.length];
+            System.arraycopy(data, headerBytes.length, newData, 0,
+                    newData.length);
+            data = newData;
+        }
+        return data;
+    }
+}

edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java

@@ -20,11 +20,7 @@
 package com.raytheon.uf.edex.plugin.manualIngest;
 
 import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
-import java.io.InputStream;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
 import org.apache.camel.Exchange;
 import org.apache.camel.Processor;
@@ -33,6 +29,7 @@ import org.springframework.util.FileCopyUtils;
 import com.raytheon.uf.common.status.IUFStatusHandler;
 import com.raytheon.uf.common.status.UFStatus;
 import com.raytheon.uf.common.status.UFStatus.Priority;
+import com.raytheon.uf.common.util.header.WMOHeaderFinder;
 import com.raytheon.uf.edex.core.EDEXUtil;
 import com.raytheon.uf.edex.core.EdexException;
 import com.raytheon.uf.edex.core.props.PropertiesFactory;
@@ -64,10 +61,6 @@ public class MessageGenerator implements Processor {
             .getEnvProperties().getEnvValue("ARCHIVEDIR")
             + File.separator + "manual";
 
-    private static Pattern WMOPATTERN = Pattern
-            .compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
-                    + "\\d{6}[^\\r\\n]*)[\\r\\n]+");
-
     private static MessageGenerator instance = new MessageGenerator();
 
     private String ingestRoute = null;
@@ -91,39 +84,17 @@ public class MessageGenerator implements Processor {
      */
     @Override
     public void process(Exchange arg0) throws Exception {
-
-        byte[] header = new byte[100];
-
         File file = (File) arg0.getIn().getBody();
-
         if (file != null) {
             String fileName = file.getName();
-
+            String messageHeader = WMOHeaderFinder.find(file);
-            InputStream is = null;
+            if (messageHeader == null) {
-            try {
+                messageHeader = fileName;
-                is = new FileInputStream(file);
-                is.read(header);
-                String sHeader = new String(header, "ISO-8859-1");
-                Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
-                String messageHeader = fileName;
-
-                if (wmoSearch.find()) {
-                    messageHeader = wmoSearch.group();
-                }
-
-                arg0.getIn().setBody(file.toString());
-                arg0.getIn().setHeader("header", messageHeader);
-                arg0.getIn().setHeader("enqueueTime",
-                        System.currentTimeMillis());
-            } finally {
-                if (is != null) {
-                    try {
-                        is.close();
-                    } catch (IOException e) {
-                        // ignore
-                    }
-                }
             }
+
+            arg0.getIn().setBody(file.toString());
+            arg0.getIn().setHeader("header", messageHeader);
+            arg0.getIn().setHeader("enqueueTime", System.currentTimeMillis());
         } else {
             // No file received
             arg0.getOut().setFault(true);
@@ -135,7 +106,7 @@ public class MessageGenerator implements Processor {
 
         // Determine the sub-directory
         String inputPath = inFile.getParent();
-        
+
         // Split on the manual directory to get the sub-directory
         String[] parts = inputPath.split("manual");
         File dir = null;
@@ -144,7 +115,7 @@ public class MessageGenerator implements Processor {
         } else {
             dir = new File(path);
         }
-        
+
         if (!dir.exists()) {
             dir.mkdirs();
         }
@@ -153,7 +124,8 @@ public class MessageGenerator implements Processor {
 
         try {
             FileCopyUtils.copy(inFile, newFile);
-            statusHandler.handle(Priority.INFO, "DataManual: " + inFile.getAbsolutePath());
+            statusHandler.handle(Priority.INFO,
+                    "DataManual: " + inFile.getAbsolutePath());
         } catch (IOException e) {
             statusHandler.handle(Priority.ERROR, "Failed to copy file ["
                     + inFile.getAbsolutePath() + "] to archive dir", e);