Issue #2838 Archive binary feeder creation.
Change-Id: I4e431d62e3d27565af75d52012fe042a18fd1f9c Former-commit-id:ac03675880[formerly9b7715e5e0] [formerly995d962a66[formerly 16c38960863df2f993c6ee7e18de31824cbdbb08]] Former-commit-id:995d962a66Former-commit-id:03843090fa
This commit is contained in:
Roger Ferrel
parent
29f9a87085
commit
dae52ad3a0
14 changed files with 524 additions and 0 deletions
|
|
@ -72,6 +72,13 @@
|
|||
id="com.raytheon.uf.edex.archive.feature"
|
||||
version="0.0.0"/>
|
||||
|
||||
<!-- Comment out this include before checking in. This plugin only for debugging. -->
|
||||
<!--
|
||||
<includes
|
||||
id="com.raytheon.uf.edex.archive.feeder.feature"
|
||||
version="0.0.0"/>
|
||||
-->
|
||||
|
||||
<includes
|
||||
id="com.raytheon.uf.edex.text.feature"
|
||||
version="0.0.0"/>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>com.raytheon.uf.edex.archive.feeder.feature</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.pde.FeatureBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.pde.FeatureNature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -0,0 +1 @@
|
|||
bin.includes = feature.xml
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feature
|
||||
id="com.raytheon.uf.edex.archive.feeder.feature"
|
||||
label="Archive Binary Feeder Feature"
|
||||
version="1.0.0.qualifier"
|
||||
provider-name="RAYTHEON">
|
||||
|
||||
<description url="http://www.example.com/description">
|
||||
Use for testing of archive binary files. Not part of standard EDEX.
|
||||
</description>
|
||||
|
||||
<copyright url="http://www.example.com/copyright">
|
||||
[Enter Copyright Description here.]
|
||||
</copyright>
|
||||
|
||||
<license url="http://www.example.com/license">
|
||||
[Enter License Description here.]
|
||||
</license>
|
||||
|
||||
<plugin
|
||||
id="com.raytheon.uf.edex.archive.feeder"
|
||||
download-size="0"
|
||||
install-size="0"
|
||||
version="0.0.0"
|
||||
unpack="false"/>
|
||||
|
||||
</feature>
|
||||
7
edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath
Normal file
7
edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<classpath>
|
||||
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7"/>
|
||||
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
|
||||
<classpathentry kind="src" path="src"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
28
edexOsgi/com.raytheon.uf.edex.archive.feeder/.project
Normal file
28
edexOsgi/com.raytheon.uf.edex.archive.feeder/.project
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<projectDescription>
|
||||
<name>com.raytheon.uf.edex.archive.feeder</name>
|
||||
<comment></comment>
|
||||
<projects>
|
||||
</projects>
|
||||
<buildSpec>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.jdt.core.javabuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.pde.ManifestBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
<buildCommand>
|
||||
<name>org.eclipse.pde.SchemaBuilder</name>
|
||||
<arguments>
|
||||
</arguments>
|
||||
</buildCommand>
|
||||
</buildSpec>
|
||||
<natures>
|
||||
<nature>org.eclipse.pde.PluginNature</nature>
|
||||
<nature>org.eclipse.jdt.core.javanature</nature>
|
||||
</natures>
|
||||
</projectDescription>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
eclipse.preferences.version=1
|
||||
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
|
||||
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
|
||||
org.eclipse.jdt.core.compiler.compliance=1.7
|
||||
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
|
||||
org.eclipse.jdt.core.compiler.source=1.7
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
Manifest-Version: 1.0
|
||||
Bundle-ManifestVersion: 2
|
||||
Bundle-Name: Utility
|
||||
Bundle-SymbolicName: com.raytheon.uf.edex.archive.feeder
|
||||
Bundle-Version: 1.0.0.qualifier
|
||||
Bundle-Activator: com.raytheon.uf.edex.archive.feeder.Activator
|
||||
Bundle-Vendor: RAYTHEON
|
||||
Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.12.1174",
|
||||
com.raytheon.edex.common;bundle-version="1.12.1174",
|
||||
com.raytheon.uf.edex.database;bundle-version="1.0.0",
|
||||
org.eclipse.ui,
|
||||
org.eclipse.core.runtime,
|
||||
com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174",
|
||||
com.raytheon.uf.common.status;bundle-version="1.12.1174",
|
||||
org.apache.camel;bundle-version="1.0.0",
|
||||
com.raytheon.uf.edex.site;bundle-version="1.0.0",
|
||||
com.raytheon.uf.edex.core;bundle-version="1.12.1174",
|
||||
org.apache.commons.beanutils;bundle-version="1.8.3"
|
||||
Bundle-RequiredExecutionEnvironment: JavaSE-1.7
|
||||
Bundle-ActivationPolicy: lazy
|
||||
20
edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt
Normal file
20
edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
The com.raytheon.uf.edex.archive.feeder plugin is not part of the standard EDEX deploy.
|
||||
It is a tool for debugging the binary files generated for archiving.
|
||||
|
||||
The easiest way to generate this plugin jar is to add the following to
|
||||
com.raytheon.edex.feature.uframe/feature.xml:
|
||||
|
||||
<includes
|
||||
id="com.raytheon.uf.edex.archive.feeder.feature"
|
||||
version="0.0.0"/>
|
||||
|
||||
Then run the build.edex deploy-install.xml. This will create the plugin jar file:
|
||||
/awips2/edex/lib/plugins/com.raytheon.uf.edex.archive.feeder.jar
|
||||
|
||||
the needed configuration file:
|
||||
|
||||
/awips2/edex/conf/resources/com.raytheon.uf.edex.archive.feeder.properties
|
||||
|
||||
By placing these two files in the corresponding locations on the desired EDEX server and
|
||||
restarting EDEX will start the clusteredArchiveBinaryFeederProc. You can then drop the
|
||||
binary files into the archive.feeder.directory, default /tmp/archive-feeder.
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
source.. = src/
|
||||
output.. = bin/
|
||||
bin.includes = META-INF/,\
|
||||
.,\
|
||||
res/
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
|
||||
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
|
||||
<camelContext id="clusteredArchiveBinaryFeederProc"
|
||||
xmlns="http://camel.apache.org/schema/spring"
|
||||
errorHandlerRef="errorHandler">
|
||||
<endpoint id="abfFileEndpoint"
|
||||
uri="file:${archive.feeder.directory}?delete=true&delay=5000&maxMessagesPerPoll=1000&exclusiveReadLockStrategy=#abfFileChangedStrategy" />
|
||||
|
||||
<route id="abfFileScan">
|
||||
<from ref="abfFileEndpoint" />
|
||||
<bean ref="abfProc" method="processEvent" />
|
||||
</route>
|
||||
</camelContext>
|
||||
|
||||
<bean id="abfFileChangedStrategy" class="com.raytheon.uf.edex.esb.camel.FileChangedExclusiveReadLockStrategy"/>
|
||||
<bean id="abfProc" class="com.raytheon.uf.edex.archive.feeder.ArchiveBinaryFeederIngester"/>
|
||||
</beans>
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
# When true enables the archive binary feeder
|
||||
archive.feeder.enable=true
|
||||
|
||||
# Directory to check for binary files to ingest.
|
||||
archive.feeder.directory=/tmp/archive-feeder
|
||||
|
|
@ -0,0 +1,249 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.edex.archive.feeder;
|
||||
|
||||
import java.io.File;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.persistence.ManyToOne;
|
||||
|
||||
import org.apache.commons.beanutils.PropertyUtils;
|
||||
|
||||
import com.raytheon.uf.common.dataplugin.PluginDataObject;
|
||||
import com.raytheon.uf.common.dataplugin.PluginException;
|
||||
import com.raytheon.uf.common.status.IUFStatusHandler;
|
||||
import com.raytheon.uf.common.status.UFStatus;
|
||||
import com.raytheon.uf.common.status.UFStatus.Priority;
|
||||
import com.raytheon.uf.edex.core.EdexException;
|
||||
import com.raytheon.uf.edex.database.plugin.PluginDao;
|
||||
import com.raytheon.uf.edex.database.plugin.PluginFactory;
|
||||
|
||||
/**
|
||||
* This class used to process binary files created for the archive directory.
|
||||
* Not intended for production. Designed for testing of problems with creating
|
||||
* archive data.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* Feb 25, 2014 2838 rferrel Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author rferrel
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public class ArchiveBinaryFeederIngester {
|
||||
|
||||
protected static final IUFStatusHandler statusHandler = UFStatus
|
||||
.getHandler(ArchiveBinaryFeederIngester.class);
|
||||
|
||||
/** The ManyToOne fields in a given class. */
|
||||
private final Map<Class<?>, Field[]> manyToOneFieldMap = new HashMap<Class<?>, Field[]>();
|
||||
|
||||
/**
|
||||
* The constructor. if needed attempts to created the drop directory
|
||||
*/
|
||||
public ArchiveBinaryFeederIngester() {
|
||||
String rootName = System.getProperty("archive.feeder.directory");
|
||||
if (rootName != null) {
|
||||
File rootDir = new File(rootName);
|
||||
if (rootDir.exists()) {
|
||||
if (!rootDir.isDirectory()) {
|
||||
if (rootDir.delete()) {
|
||||
rootDir.mkdirs();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
rootDir.mkdirs();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process archive binary file.
|
||||
*
|
||||
* @param file
|
||||
* @throws EdexException
|
||||
*/
|
||||
public void processEvent(File file) throws EdexException {
|
||||
IUFStatusHandler statusHandler = UFStatus
|
||||
.getHandler(ArchiveBinaryFeederIngester.class);
|
||||
|
||||
boolean enabled = Boolean.getBoolean("archive.feeder.enable");
|
||||
if (!enabled) {
|
||||
if (statusHandler.isPriorityEnabled(Priority.WARN)) {
|
||||
StringBuilder sb = new StringBuilder(
|
||||
ArchiveBinaryFeederIngester.class.getName());
|
||||
sb.setLength(sb.lastIndexOf("."));
|
||||
statusHandler
|
||||
.handle(Priority.WARN,
|
||||
String.format(
|
||||
"file \"%s\" not processed; feeder not enabled consider removing plugin: %s",
|
||||
file, sb.toString()));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
ArchiveBinaryReader reader = new ArchiveBinaryReader(file);
|
||||
|
||||
try {
|
||||
if (statusHandler.isPriorityEnabled(Priority.INFO)) {
|
||||
statusHandler.handle(
|
||||
Priority.INFO,
|
||||
String.format("Start processing file \"%1$s\".",
|
||||
file.getAbsoluteFile()));
|
||||
}
|
||||
|
||||
Object object = reader.getObject();
|
||||
if (!(object instanceof List<?>)) {
|
||||
if (statusHandler.isPriorityEnabled(Priority.INFO)) {
|
||||
statusHandler
|
||||
.info(String
|
||||
.format("file: \"%s\" does not contain a list of PluginDataObjects",
|
||||
file.getAbsolutePath()));
|
||||
}
|
||||
} else {
|
||||
List<?> objects = (List<?>) object;
|
||||
process(objects);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (statusHandler.isPriorityEnabled(Priority.PROBLEM)) {
|
||||
statusHandler.handle(Priority.PROBLEM, String.format(
|
||||
"Problem processing the file: \"%s\", %s",
|
||||
file.getAbsolutePath(), e.getLocalizedMessage()));
|
||||
}
|
||||
|
||||
} finally {
|
||||
if (statusHandler.isPriorityEnabled(Priority.INFO)) {
|
||||
long time = System.currentTimeMillis() - start;
|
||||
statusHandler.handle(Priority.INFO, String.format(
|
||||
"Finished processing file \"%1$s\" in %2$d msec.",
|
||||
file.getAbsoluteFile(), time));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks to see if the objects is a list of PluginDataObjects and attempts
|
||||
* use the proper dao to send list to the data base.
|
||||
*
|
||||
* @param objects
|
||||
* @throws PluginException
|
||||
*/
|
||||
private void process(List<?> objects) throws PluginException {
|
||||
if (objects.size() > 0) {
|
||||
Object o = objects.iterator().next();
|
||||
|
||||
if (o instanceof PluginDataObject) {
|
||||
PluginDataObject pdo = (PluginDataObject) o;
|
||||
String pdoName = pdo.getPluginName();
|
||||
|
||||
PluginDao dao = PluginFactory.getInstance().getPluginDao(
|
||||
pdoName);
|
||||
|
||||
processManytoOneFields(objects, pdo.getClass(), dao);
|
||||
|
||||
dao.persistAll(objects);
|
||||
} else {
|
||||
throw new PluginException("Not a list of PluginDataObjects.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the fields in the class that are an instance of the ManyToOne
|
||||
* class. This will always return a non null result.
|
||||
*
|
||||
* @param clazz
|
||||
* @return fields
|
||||
*/
|
||||
private synchronized Field[] getManyToOneFields(Class<?> clazz) {
|
||||
Field[] fields = manyToOneFieldMap.get(clazz);
|
||||
if (fields == null) {
|
||||
List<Field> fieldList = new ArrayList<Field>();
|
||||
Class<?> currentClass = clazz;
|
||||
|
||||
while (currentClass != null) {
|
||||
for (Field field : currentClass.getDeclaredFields()) {
|
||||
if (field.getAnnotation(ManyToOne.class) != null) {
|
||||
fieldList.add(field);
|
||||
}
|
||||
}
|
||||
currentClass = currentClass.getSuperclass();
|
||||
}
|
||||
|
||||
fields = fieldList.toArray(new Field[fieldList.size()]);
|
||||
manyToOneFieldMap.put(clazz, fields);
|
||||
}
|
||||
|
||||
return fields;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the Many to one fields of a class so their values are in the data
|
||||
* base prior to attempting to place the objects in the data base.
|
||||
*
|
||||
* @param objects
|
||||
* @param clazz
|
||||
*/
|
||||
private void processManytoOneFields(Collection<?> objects, Class<?> clazz,
|
||||
PluginDao dao) {
|
||||
Field[] fields = getManyToOneFields(clazz);
|
||||
if (fields.length > 0) {
|
||||
Map<Object, Object> foMap = new HashMap<Object, Object>();
|
||||
for (Field field : fields) {
|
||||
for (Object object : objects) {
|
||||
try {
|
||||
Object fieldObject = PropertyUtils.getProperty(object,
|
||||
field.getName());
|
||||
if (fieldObject != null) {
|
||||
Object fo = foMap.get(fieldObject);
|
||||
if (fo == null) {
|
||||
foMap.put(fieldObject, fieldObject);
|
||||
} else {
|
||||
// Set to the instance being sent to the dao.
|
||||
PropertyUtils.setProperty(object,
|
||||
field.getName(), fo);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
statusHandler.handle(Priority.PROBLEM,
|
||||
e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
Collection<?> fieldObjects = foMap.values();
|
||||
processManytoOneFields(fieldObjects, field.getType(), dao);
|
||||
dao.persistAll(fieldObjects);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* This software was developed and / or modified by Raytheon Company,
|
||||
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
|
||||
*
|
||||
* U.S. EXPORT CONTROLLED TECHNICAL DATA
|
||||
* This software product contains export-restricted data whose
|
||||
* export/transfer/disclosure is restricted by U.S. law. Dissemination
|
||||
* to non-U.S. persons whether in the United States or abroad requires
|
||||
* an export license or other authorization.
|
||||
*
|
||||
* Contractor Name: Raytheon Company
|
||||
* Contractor Address: 6825 Pine Street, Suite 340
|
||||
* Mail Stop B8
|
||||
* Omaha, NE 68106
|
||||
* 402.291.0100
|
||||
*
|
||||
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
|
||||
* further licensing information.
|
||||
**/
|
||||
package com.raytheon.uf.edex.archive.feeder;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
|
||||
import com.raytheon.uf.common.serialization.DynamicSerializationManager;
|
||||
import com.raytheon.uf.common.serialization.DynamicSerializationManager.SerializationType;
|
||||
import com.raytheon.uf.common.serialization.SerializationException;
|
||||
|
||||
/**
|
||||
* This class uses the DynamicSerializationManager to create an instance of the
|
||||
* first serialized object in a file.
|
||||
*
|
||||
* <pre>
|
||||
*
|
||||
* SOFTWARE HISTORY
|
||||
*
|
||||
* Date Ticket# Engineer Description
|
||||
* ------------ ---------- ----------- --------------------------
|
||||
* Feb 25, 2014 2838 rferrel Initial creation
|
||||
*
|
||||
* </pre>
|
||||
*
|
||||
* @author rferrel
|
||||
* @version 1.0
|
||||
*/
|
||||
|
||||
public class ArchiveBinaryReader {
|
||||
|
||||
/** File containing the serialized object. */
|
||||
private File binFile;
|
||||
|
||||
/** Instance of the file's serialized object. */
|
||||
Object object;
|
||||
|
||||
/**
|
||||
* Constructor
|
||||
*
|
||||
* @param binFile
|
||||
* - File containing serialized object.
|
||||
*/
|
||||
public ArchiveBinaryReader(File binFile) {
|
||||
this.binFile = binFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* The object obtained from the binFile.
|
||||
*
|
||||
* @return object
|
||||
* @throws FileNotFoundException
|
||||
* @throws SerializationException
|
||||
*/
|
||||
public Object getObject() throws FileNotFoundException,
|
||||
SerializationException {
|
||||
if (object == null) {
|
||||
object = readObject();
|
||||
}
|
||||
return object;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an instance of the file's serialized object.
|
||||
*
|
||||
* @return object
|
||||
* @throws FileNotFoundException
|
||||
* @throws SerializationException
|
||||
*/
|
||||
private Object readObject() throws FileNotFoundException,
|
||||
SerializationException {
|
||||
InputStream inputStream = null;
|
||||
Object object = null;
|
||||
try {
|
||||
DynamicSerializationManager dsm = DynamicSerializationManager
|
||||
.getManager(SerializationType.Thrift);
|
||||
inputStream = new BufferedInputStream(new FileInputStream(binFile));
|
||||
object = dsm.deserialize(inputStream);
|
||||
} finally {
|
||||
if (inputStream != null) {
|
||||
try {
|
||||
inputStream.close();
|
||||
} catch (IOException e) {
|
||||
// Ignore
|
||||
}
|
||||
inputStream = null;
|
||||
}
|
||||
}
|
||||
return object;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue