xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #626 Moved data unzipper and wmo header remover to common so it can potentially be used by GOES-R" into development

Browse source 
Former-commit-id: 1de59b48cf [formerly 9e3cd2546a] [formerly 77c2ab2c05 [formerly 39e74c7e2aa9f78938c85d0433decb89f4ef1325]]
Former-commit-id: 77c2ab2c05
Former-commit-id: d131cd42cf
This commit is contained in:
Richard Peter 2012-07-27 14:50:00 -05:00 • committed by Gerrit Code Review
commit b6a1b5011f
6 changed files with 284 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
edexOsgi/build.edex/esb/conf/spring/edex.xml
View file

@ -154,7 +154,9 @@
<constructor-arg><ref bean="pluginSetup"/></constructor-arg>
</bean>
<bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>
<bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>
<bean id="extractWMOHeader" class="com.raytheon.uf.common.util.header.WMOHeaderRemover"/>
<bean id="dataUnzipper" class="com.raytheon.uf.common.util.DataUnzipper"/>
<bean id="errorHandler" class="org.apache.camel.builder.LoggingErrorHandlerBuilder"/>

1
edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF
View file

@ -8,5 +8,6 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Require-Bundle: org.junit;bundle-version="1.0.0"
Export-Package: com.raytheon.uf.common.util,
com.raytheon.uf.common.util.cache,
com.raytheon.uf.common.util.header,
com.raytheon.uf.common.util.registry
Import-Package: org.apache.commons.lang

94
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java Normal file
View file

@ -0,0 +1,94 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipInputStream;
/**
* Bean that will attempt to decompress a byte[]. Will throw IOException if data
* can not be unzipped
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 17, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class DataUnzipper {
/**
* Uses ZLIB decompression to unzip the data
*
* @param data
* @return
* @throws IOException
*/
public byte[] unzip(byte[] data) throws IOException {
return unzip(new ZipInputStream(new ByteArrayInputStream(data)),
data.length * 2);
}
/**
* Uses GZIP decompression to unzip the data
*
* @param data
* @return
* @throws IOException
*/
public byte[] gunzip(byte[] data) throws IOException {
return unzip(new GZIPInputStream(new ByteArrayInputStream(data)),
data.length * 2);
}
/**
* Unzips the {@link InputStream} into a byte[] this function will close the
* in stream after decompression
*
* @param in
* @param size
* @return
* @throws IOException
*/
private byte[] unzip(InputStream in, int size) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(size);
try {
byte[] buf = new byte[4096];
int len;
while ((len = in.read(buf)) > 0)
out.write(buf, 0, len);
} finally {
in.close();
}
return out.toByteArray();
}
}

102
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java Normal file
View file

@ -0,0 +1,102 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util.header;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Utility class for finding a WMO header
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 27, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class WMOHeaderFinder {
private static Pattern WMOPATTERN = Pattern
.compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
+ "\\d{6}[^\\r\\n]*)[\\r\\n]+");
/**
* Finds and returns the WMO header on the {@link File}
*
* @param file
* @return
* @throws FileNotFoundException
* @throws IOException
*/
public static String find(File file) throws FileNotFoundException,
IOException {
return find(new FileInputStream(file));
}
/**
* Finds and returns the WMO header on the data
*
* @param data
* @return
* @throws IOException
*/
public static String find(byte[] data) throws IOException {
return find(new ByteArrayInputStream(data));
}
/**
* Finds and returns the WMO header from the {@link InputStream}. Didn't
* expose this function because I wanted to ensure that it was responsible
* for closing the input stream
*
* @param in
* @return
* @throws IOException
*/
private static String find(InputStream in) throws IOException {
try {
byte[] header = new byte[100];
in.read(header);
String sHeader = new String(header, "ISO-8859-1");
Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
String messageHeader = null;
if (wmoSearch.find()) {
messageHeader = wmoSearch.group();
}
return messageHeader;
} finally {
in.close();
}
}
}

72
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java Normal file
View file

@ -0,0 +1,72 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util.header;
import java.io.IOException;
/**
* Bean that will attempt to extract the WMO header found in the {@link Headers}
* argument from the begining of the byte[]
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 17, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class WMOHeaderRemover {
public byte[] remove(byte[] data) throws IOException {
String header = WMOHeaderFinder.find(data);
if (header == null) {
// No header found, nothing to do
return data;
}
byte[] headerBytes = header.getBytes();
boolean equal = false;
// Do < since there should be something after the header
if (headerBytes.length < data.length) {
equal = true;
for (int i = 0; i < headerBytes.length; ++i) {
if (headerBytes[i] != data[i]) {
equal = false;
break;
}
}
}
if (equal) {
byte[] newData = new byte[data.length - headerBytes.length];
System.arraycopy(data, headerBytes.length, newData, 0,
newData.length);
data = newData;
}
return data;
}
}

52
edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java
View file

@ -20,11 +20,7 @@
package com.raytheon.uf.edex.plugin.manualIngest;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@ -33,6 +29,7 @@ import org.springframework.util.FileCopyUtils;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.util.header.WMOHeaderFinder;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.EdexException;
import com.raytheon.uf.edex.core.props.PropertiesFactory;
@ -64,10 +61,6 @@ public class MessageGenerator implements Processor {
.getEnvProperties().getEnvValue("ARCHIVEDIR")
+ File.separator + "manual";
private static Pattern WMOPATTERN = Pattern
.compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
+ "\\d{6}[^\\r\\n]*)[\\r\\n]+");
private static MessageGenerator instance = new MessageGenerator();
private String ingestRoute = null;
@ -91,39 +84,17 @@ public class MessageGenerator implements Processor {
*/
@Override
public void process(Exchange arg0) throws Exception {
byte[] header = new byte[100];
File file = (File) arg0.getIn().getBody();
if (file != null) {
String fileName = file.getName();
InputStream is = null;
try {
is = new FileInputStream(file);
is.read(header);
String sHeader = new String(header, "ISO-8859-1");
Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
String messageHeader = fileName;
if (wmoSearch.find()) {
messageHeader = wmoSearch.group();
}
arg0.getIn().setBody(file.toString());
arg0.getIn().setHeader("header", messageHeader);
arg0.getIn().setHeader("enqueueTime",
System.currentTimeMillis());
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
// ignore
}
}
String messageHeader = WMOHeaderFinder.find(file);
if (messageHeader == null) {
messageHeader = fileName;
}
arg0.getIn().setBody(file.toString());
arg0.getIn().setHeader("header", messageHeader);
arg0.getIn().setHeader("enqueueTime", System.currentTimeMillis());
} else {
// No file received
arg0.getOut().setFault(true);
@ -135,7 +106,7 @@ public class MessageGenerator implements Processor {
// Determine the sub-directory
String inputPath = inFile.getParent();
// Split on the manual directory to get the sub-directory
String[] parts = inputPath.split("manual");
File dir = null;
@ -144,7 +115,7 @@ public class MessageGenerator implements Processor {
} else {
dir = new File(path);
}
if (!dir.exists()) {
dir.mkdirs();
}
@ -153,7 +124,8 @@ public class MessageGenerator implements Processor {
try {
FileCopyUtils.copy(inFile, newFile);
statusHandler.handle(Priority.INFO, "DataManual: " + inFile.getAbsolutePath());
statusHandler.handle(Priority.INFO,
"DataManual: " + inFile.getAbsolutePath());
} catch (IOException e) {
statusHandler.handle(Priority.ERROR, "Failed to copy file ["
+ inFile.getAbsolutePath() + "] to archive dir", e);