xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #626 Moved data unzipper and wmo header remover to common so it can potentially be used by GOES-R

Browse source 
Amend: Added WMO header finder since the header property will have additional information in it when file comes from LDM.  Also changed MessageGenerator to use new class for finding the wmo header

Amend: Added try-finally for input stream in DataUnzipper

Change-Id: I4ece2f5f24a429aa41969f51040a58e67e79e1c1

Former-commit-id: 2f17b2f49f65da4b7b7fd531377873cb9eb941dd
This commit is contained in:
Max Schenkelberg 2012-07-26 17:37:07 -05:00
parent dbe185f3a0
commit b669f57ee4
6 changed files with 284 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
edexOsgi/build.edex/esb/conf/spring/edex.xml
View file

@ -154,7 +154,9 @@
<constructor-arg><ref bean="pluginSetup"/></constructor-arg>
</bean>
<bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>
<bean id="stringToFile" class="com.raytheon.uf.edex.esb.camel.StringToFile"/>
<bean id="extractWMOHeader" class="com.raytheon.uf.common.util.header.WMOHeaderRemover"/>
<bean id="dataUnzipper" class="com.raytheon.uf.common.util.DataUnzipper"/>
<bean id="errorHandler" class="org.apache.camel.builder.LoggingErrorHandlerBuilder"/>

1
edexOsgi/com.raytheon.uf.common.util/META-INF/MANIFEST.MF
View file

@ -8,5 +8,6 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Require-Bundle: org.junit;bundle-version="1.0.0"
Export-Package: com.raytheon.uf.common.util,
com.raytheon.uf.common.util.cache,
com.raytheon.uf.common.util.header,
com.raytheon.uf.common.util.registry
Import-Package: org.apache.commons.lang

94
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/DataUnzipper.java Normal file
View file

@ -0,0 +1,94 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipInputStream;
/**
* Bean that will attempt to decompress a byte[]. Will throw IOException if data
* can not be unzipped
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 17, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class DataUnzipper {
/**
* Uses ZLIB decompression to unzip the data
*
* @param data
* @return
* @throws IOException
*/
public byte[] unzip(byte[] data) throws IOException {
return unzip(new ZipInputStream(new ByteArrayInputStream(data)),
data.length * 2);
}
/**
* Uses GZIP decompression to unzip the data
*
* @param data
* @return
* @throws IOException
*/
public byte[] gunzip(byte[] data) throws IOException {
return unzip(new GZIPInputStream(new ByteArrayInputStream(data)),
data.length * 2);
}
/**
* Unzips the {@link InputStream} into a byte[] this function will close the
* in stream after decompression
*
* @param in
* @param size
* @return
* @throws IOException
*/
private byte[] unzip(InputStream in, int size) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream(size);
try {
byte[] buf = new byte[4096];
int len;
while ((len = in.read(buf)) > 0)
out.write(buf, 0, len);
} finally {
in.close();
}
return out.toByteArray();
}
}

102
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderFinder.java Normal file
View file

@ -0,0 +1,102 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util.header;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Utility class for finding a WMO header
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 27, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class WMOHeaderFinder {
private static Pattern WMOPATTERN = Pattern
.compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
+ "\\d{6}[^\\r\\n]*)[\\r\\n]+");
/**
* Finds and returns the WMO header on the {@link File}
*
* @param file
* @return
* @throws FileNotFoundException
* @throws IOException
*/
public static String find(File file) throws FileNotFoundException,
IOException {
return find(new FileInputStream(file));
}
/**
* Finds and returns the WMO header on the data
*
* @param data
* @return
* @throws IOException
*/
public static String find(byte[] data) throws IOException {
return find(new ByteArrayInputStream(data));
}
/**
* Finds and returns the WMO header from the {@link InputStream}. Didn't
* expose this function because I wanted to ensure that it was responsible
* for closing the input stream
*
* @param in
* @return
* @throws IOException
*/
private static String find(InputStream in) throws IOException {
try {
byte[] header = new byte[100];
in.read(header);
String sHeader = new String(header, "ISO-8859-1");
Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
String messageHeader = null;
if (wmoSearch.find()) {
messageHeader = wmoSearch.group();
}
return messageHeader;
} finally {
in.close();
}
}
}

72
edexOsgi/com.raytheon.uf.common.util/src/com/raytheon/uf/common/util/header/WMOHeaderRemover.java Normal file
View file

@ -0,0 +1,72 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.util.header;
import java.io.IOException;
/**
* Bean that will attempt to extract the WMO header found in the {@link Headers}
* argument from the begining of the byte[]
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jul 17, 2012 mschenke Initial creation
*
* </pre>
*
* @author mschenke
* @version 1.0
*/
public class WMOHeaderRemover {
public byte[] remove(byte[] data) throws IOException {
String header = WMOHeaderFinder.find(data);
if (header == null) {
// No header found, nothing to do
return data;
}
byte[] headerBytes = header.getBytes();
boolean equal = false;
// Do < since there should be something after the header
if (headerBytes.length < data.length) {
equal = true;
for (int i = 0; i < headerBytes.length; ++i) {
if (headerBytes[i] != data[i]) {
equal = false;
break;
}
}
}
if (equal) {
byte[] newData = new byte[data.length - headerBytes.length];
System.arraycopy(data, headerBytes.length, newData, 0,
newData.length);
data = newData;
}
return data;
}
}

52
edexOsgi/com.raytheon.uf.edex.plugin.manualIngest/src/com/raytheon/uf/edex/plugin/manualIngest/MessageGenerator.java
View file

@ -20,11 +20,7 @@
package com.raytheon.uf.edex.plugin.manualIngest;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.camel.Exchange;
import org.apache.camel.Processor;
@ -33,6 +29,7 @@ import org.springframework.util.FileCopyUtils;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.util.header.WMOHeaderFinder;
import com.raytheon.uf.edex.core.EDEXUtil;
import com.raytheon.uf.edex.core.EdexException;
import com.raytheon.uf.edex.core.props.PropertiesFactory;
@ -64,10 +61,6 @@ public class MessageGenerator implements Processor {
.getEnvProperties().getEnvValue("ARCHIVEDIR")
+ File.separator + "manual";
private static Pattern WMOPATTERN = Pattern
.compile("([A-Z]{3}[A-Z0-9](\\d{0,2}|[A-Z]{0,2}) [A-Z0-9 ]{4} "
+ "\\d{6}[^\\r\\n]*)[\\r\\n]+");
private static MessageGenerator instance = new MessageGenerator();
private String ingestRoute = null;
@ -91,39 +84,17 @@ public class MessageGenerator implements Processor {
*/
@Override
public void process(Exchange arg0) throws Exception {
byte[] header = new byte[100];
File file = (File) arg0.getIn().getBody();
if (file != null) {
String fileName = file.getName();
InputStream is = null;
try {
is = new FileInputStream(file);
is.read(header);
String sHeader = new String(header, "ISO-8859-1");
Matcher wmoSearch = WMOPATTERN.matcher(sHeader);
String messageHeader = fileName;
if (wmoSearch.find()) {
messageHeader = wmoSearch.group();
}
arg0.getIn().setBody(file.toString());
arg0.getIn().setHeader("header", messageHeader);
arg0.getIn().setHeader("enqueueTime",
System.currentTimeMillis());
} finally {
if (is != null) {
try {
is.close();
} catch (IOException e) {
// ignore
}
}
String messageHeader = WMOHeaderFinder.find(file);
if (messageHeader == null) {
messageHeader = fileName;
}
arg0.getIn().setBody(file.toString());
arg0.getIn().setHeader("header", messageHeader);
arg0.getIn().setHeader("enqueueTime", System.currentTimeMillis());
} else {
// No file received
arg0.getOut().setFault(true);
@ -135,7 +106,7 @@ public class MessageGenerator implements Processor {
// Determine the sub-directory
String inputPath = inFile.getParent();
// Split on the manual directory to get the sub-directory
String[] parts = inputPath.split("manual");
File dir = null;
@ -144,7 +115,7 @@ public class MessageGenerator implements Processor {
} else {
dir = new File(path);
}
if (!dir.exists()) {
dir.mkdirs();
}
@ -153,7 +124,8 @@ public class MessageGenerator implements Processor {
try {
FileCopyUtils.copy(inFile, newFile);
statusHandler.handle(Priority.INFO, "DataManual: " + inFile.getAbsolutePath());
statusHandler.handle(Priority.INFO,
"DataManual: " + inFile.getAbsolutePath());
} catch (IOException e) {
statusHandler.handle(Priority.ERROR, "Failed to copy file ["
+ inFile.getAbsolutePath() + "] to archive dir", e);