From 592bbd8241f83c3bb447f1b22564292dc3273501 Mon Sep 17 00:00:00 2001 From: Roger Ferrel Date: Tue, 25 Feb 2014 10:11:20 -0600 Subject: [PATCH] Issue #2838 Archive binary feeder creation. Change-Id: I4e431d62e3d27565af75d52012fe042a18fd1f9c Former-commit-id: 98bd4174d3e1838b3af47abada7bc5137b7b53d6 [formerly ac0367588065b8e344c60f5dbca423903afc5bbf] [formerly 9b7715e5e0d67e865081023f01e4dbe75e5e56f1] [formerly 995d962a66114ba9709e3ec350be31f0a238c830 [formerly 9b7715e5e0d67e865081023f01e4dbe75e5e56f1 [formerly 16c38960863df2f993c6ee7e18de31824cbdbb08]]] Former-commit-id: 995d962a66114ba9709e3ec350be31f0a238c830 Former-commit-id: 8b2aa969d1f1fe5980fc422623cca4e71e6d36fc [formerly 03843090fa272f5dd9fcec978f9f918b8020c87f] Former-commit-id: dae52ad3a052b0be522f882e150b880c136637b7 --- .../feature.xml | 7 + .../.project | 17 ++ .../build.properties | 1 + .../feature.xml | 27 ++ .../.classpath | 7 + .../.project | 28 ++ .../.settings/org.eclipse.jdt.core.prefs | 7 + .../META-INF/MANIFEST.MF | 20 ++ .../README.txt | 20 ++ .../build.properties | 5 + .../res/spring/archiveFeeder-spring.xml | 19 ++ ...raytheon.uf.edex.archive.feeder.properties | 5 + .../feeder/ArchiveBinaryFeederIngester.java | 249 ++++++++++++++++++ .../archive/feeder/ArchiveBinaryReader.java | 112 ++++++++ 14 files changed, 524 insertions(+) create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/.project create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/build.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/feature.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/.project create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/.settings/org.eclipse.jdt.core.prefs create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/META-INF/MANIFEST.MF create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/build.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/res/spring/archiveFeeder-spring.xml create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/resources/com.raytheon.uf.edex.archive.feeder.properties create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryFeederIngester.java create mode 100644 edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryReader.java diff --git a/edexOsgi/com.raytheon.edex.feature.uframe/feature.xml b/edexOsgi/com.raytheon.edex.feature.uframe/feature.xml index f39f236137..f84a22f7b9 100644 --- a/edexOsgi/com.raytheon.edex.feature.uframe/feature.xml +++ b/edexOsgi/com.raytheon.edex.feature.uframe/feature.xml @@ -71,6 +71,13 @@ + + + + + com.raytheon.uf.edex.archive.feeder.feature + + + + + + org.eclipse.pde.FeatureBuilder + + + + + + org.eclipse.pde.FeatureNature + + diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/build.properties b/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/build.properties new file mode 100644 index 0000000000..64f93a9f0b --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/build.properties @@ -0,0 +1 @@ +bin.includes = feature.xml diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/feature.xml b/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/feature.xml new file mode 100644 index 0000000000..deae9f808c --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder.feature/feature.xml @@ -0,0 +1,27 @@ + + + + + Use for testing of archive binary files. Not part of standard EDEX. + + + + [Enter Copyright Description here.] + + + + [Enter License Description here.] + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath new file mode 100644 index 0000000000..098194ca4b --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.classpath @@ -0,0 +1,7 @@ + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/.project b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.project new file mode 100644 index 0000000000..c49d0fc471 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.project @@ -0,0 +1,28 @@ + + + com.raytheon.uf.edex.archive.feeder + + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.pde.ManifestBuilder + + + + + org.eclipse.pde.SchemaBuilder + + + + + + org.eclipse.pde.PluginNature + org.eclipse.jdt.core.javanature + + diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/.settings/org.eclipse.jdt.core.prefs b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000000..f42de363af --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,7 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.source=1.7 diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/META-INF/MANIFEST.MF b/edexOsgi/com.raytheon.uf.edex.archive.feeder/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..f00382758c --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/META-INF/MANIFEST.MF @@ -0,0 +1,20 @@ +Manifest-Version: 1.0 +Bundle-ManifestVersion: 2 +Bundle-Name: Utility +Bundle-SymbolicName: com.raytheon.uf.edex.archive.feeder +Bundle-Version: 1.0.0.qualifier +Bundle-Activator: com.raytheon.uf.edex.archive.feeder.Activator +Bundle-Vendor: RAYTHEON +Require-Bundle: com.raytheon.uf.common.serialization;bundle-version="1.12.1174", + com.raytheon.edex.common;bundle-version="1.12.1174", + com.raytheon.uf.edex.database;bundle-version="1.0.0", + org.eclipse.ui, + org.eclipse.core.runtime, + com.raytheon.uf.common.dataplugin;bundle-version="1.12.1174", + com.raytheon.uf.common.status;bundle-version="1.12.1174", + org.apache.camel;bundle-version="1.0.0", + com.raytheon.uf.edex.site;bundle-version="1.0.0", + com.raytheon.uf.edex.core;bundle-version="1.12.1174", + org.apache.commons.beanutils;bundle-version="1.8.3" +Bundle-RequiredExecutionEnvironment: JavaSE-1.7 +Bundle-ActivationPolicy: lazy diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt b/edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt new file mode 100644 index 0000000000..62246b05d3 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/README.txt @@ -0,0 +1,20 @@ +The com.raytheon.uf.edex.archive.feeder plugin is not part of the standard EDEX deploy. +It is a tool for debugging the binary files generated for archiving. + +The easiest way to generate this plugin jar is to add the following to +com.raytheon.edex.feature.uframe/feature.xml: + + + +Then run the build.edex deploy-install.xml. This will create the plugin jar file: +/awips2/edex/lib/plugins/com.raytheon.uf.edex.archive.feeder.jar + +the needed configuration file: + +/awips2/edex/conf/resources/com.raytheon.uf.edex.archive.feeder.properties + +By placing these two files in the corresponding locations on the desired EDEX server and +restarting EDEX will start the clusteredArchiveBinaryFeederProc. You can then drop the +binary files into the archive.feeder.directory, default /tmp/archive-feeder. \ No newline at end of file diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/build.properties b/edexOsgi/com.raytheon.uf.edex.archive.feeder/build.properties new file mode 100644 index 0000000000..5791d48d5f --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/build.properties @@ -0,0 +1,5 @@ +source.. = src/ +output.. = bin/ +bin.includes = META-INF/,\ + .,\ + res/ diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/res/spring/archiveFeeder-spring.xml b/edexOsgi/com.raytheon.uf.edex.archive.feeder/res/spring/archiveFeeder-spring.xml new file mode 100644 index 0000000000..2bbcd17a87 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/res/spring/archiveFeeder-spring.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/resources/com.raytheon.uf.edex.archive.feeder.properties b/edexOsgi/com.raytheon.uf.edex.archive.feeder/resources/com.raytheon.uf.edex.archive.feeder.properties new file mode 100644 index 0000000000..d7a4ec2f69 --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/resources/com.raytheon.uf.edex.archive.feeder.properties @@ -0,0 +1,5 @@ +# When true enables the archive binary feeder +archive.feeder.enable=true + +# Directory to check for binary files to ingest. +archive.feeder.directory=/tmp/archive-feeder diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryFeederIngester.java b/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryFeederIngester.java new file mode 100644 index 0000000000..83fdfad8cf --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryFeederIngester.java @@ -0,0 +1,249 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.archive.feeder; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.persistence.ManyToOne; + +import org.apache.commons.beanutils.PropertyUtils; + +import com.raytheon.uf.common.dataplugin.PluginDataObject; +import com.raytheon.uf.common.dataplugin.PluginException; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.common.status.UFStatus.Priority; +import com.raytheon.uf.edex.core.EdexException; +import com.raytheon.uf.edex.database.plugin.PluginDao; +import com.raytheon.uf.edex.database.plugin.PluginFactory; + +/** + * This class used to process binary files created for the archive directory. + * Not intended for production. Designed for testing of problems with creating + * archive data. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Feb 25, 2014 2838       rferrel     Initial creation
+ * 
+ *
+ * + * @author rferrel + * @version 1.0 + */ + +public class ArchiveBinaryFeederIngester { + + protected static final IUFStatusHandler statusHandler = UFStatus + .getHandler(ArchiveBinaryFeederIngester.class); + + /** The ManyToOne fields in a given class. */ + private final Map, Field[]> manyToOneFieldMap = new HashMap, Field[]>(); + + /** + * The constructor. if needed attempts to created the drop directory + */ + public ArchiveBinaryFeederIngester() { + String rootName = System.getProperty("archive.feeder.directory"); + if (rootName != null) { + File rootDir = new File(rootName); + if (rootDir.exists()) { + if (!rootDir.isDirectory()) { + if (rootDir.delete()) { + rootDir.mkdirs(); + } + } + } else { + rootDir.mkdirs(); + } + } + } + + /** + * Process archive binary file. + * + * @param file + * @throws EdexException + */ + public void processEvent(File file) throws EdexException { + IUFStatusHandler statusHandler = UFStatus + .getHandler(ArchiveBinaryFeederIngester.class); + + boolean enabled = Boolean.getBoolean("archive.feeder.enable"); + if (!enabled) { + if (statusHandler.isPriorityEnabled(Priority.WARN)) { + StringBuilder sb = new StringBuilder( + ArchiveBinaryFeederIngester.class.getName()); + sb.setLength(sb.lastIndexOf(".")); + statusHandler + .handle(Priority.WARN, + String.format( + "file \"%s\" not processed; feeder not enabled consider removing plugin: %s", + file, sb.toString())); + } + return; + } + + long start = System.currentTimeMillis(); + ArchiveBinaryReader reader = new ArchiveBinaryReader(file); + + try { + if (statusHandler.isPriorityEnabled(Priority.INFO)) { + statusHandler.handle( + Priority.INFO, + String.format("Start processing file \"%1$s\".", + file.getAbsoluteFile())); + } + + Object object = reader.getObject(); + if (!(object instanceof List)) { + if (statusHandler.isPriorityEnabled(Priority.INFO)) { + statusHandler + .info(String + .format("file: \"%s\" does not contain a list of PluginDataObjects", + file.getAbsolutePath())); + } + } else { + List objects = (List) object; + process(objects); + } + } catch (Exception e) { + if (statusHandler.isPriorityEnabled(Priority.PROBLEM)) { + statusHandler.handle(Priority.PROBLEM, String.format( + "Problem processing the file: \"%s\", %s", + file.getAbsolutePath(), e.getLocalizedMessage())); + } + + } finally { + if (statusHandler.isPriorityEnabled(Priority.INFO)) { + long time = System.currentTimeMillis() - start; + statusHandler.handle(Priority.INFO, String.format( + "Finished processing file \"%1$s\" in %2$d msec.", + file.getAbsoluteFile(), time)); + } + } + } + + /** + * Checks to see if the objects is a list of PluginDataObjects and attempts + * use the proper dao to send list to the data base. + * + * @param objects + * @throws PluginException + */ + private void process(List objects) throws PluginException { + if (objects.size() > 0) { + Object o = objects.iterator().next(); + + if (o instanceof PluginDataObject) { + PluginDataObject pdo = (PluginDataObject) o; + String pdoName = pdo.getPluginName(); + + PluginDao dao = PluginFactory.getInstance().getPluginDao( + pdoName); + + processManytoOneFields(objects, pdo.getClass(), dao); + + dao.persistAll(objects); + } else { + throw new PluginException("Not a list of PluginDataObjects."); + } + } + } + + /** + * Determine the fields in the class that are an instance of the ManyToOne + * class. This will always return a non null result. + * + * @param clazz + * @return fields + */ + private synchronized Field[] getManyToOneFields(Class clazz) { + Field[] fields = manyToOneFieldMap.get(clazz); + if (fields == null) { + List fieldList = new ArrayList(); + Class currentClass = clazz; + + while (currentClass != null) { + for (Field field : currentClass.getDeclaredFields()) { + if (field.getAnnotation(ManyToOne.class) != null) { + fieldList.add(field); + } + } + currentClass = currentClass.getSuperclass(); + } + + fields = fieldList.toArray(new Field[fieldList.size()]); + manyToOneFieldMap.put(clazz, fields); + } + + return fields; + } + + /** + * Process the Many to one fields of a class so their values are in the data + * base prior to attempting to place the objects in the data base. + * + * @param objects + * @param clazz + */ + private void processManytoOneFields(Collection objects, Class clazz, + PluginDao dao) { + Field[] fields = getManyToOneFields(clazz); + if (fields.length > 0) { + Map foMap = new HashMap(); + for (Field field : fields) { + for (Object object : objects) { + try { + Object fieldObject = PropertyUtils.getProperty(object, + field.getName()); + if (fieldObject != null) { + Object fo = foMap.get(fieldObject); + if (fo == null) { + foMap.put(fieldObject, fieldObject); + } else { + // Set to the instance being sent to the dao. + PropertyUtils.setProperty(object, + field.getName(), fo); + } + } + } catch (Exception e) { + statusHandler.handle(Priority.PROBLEM, + e.getLocalizedMessage(), e); + } + } + + Collection fieldObjects = foMap.values(); + processManytoOneFields(fieldObjects, field.getType(), dao); + dao.persistAll(fieldObjects); + } + } + } +} diff --git a/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryReader.java b/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryReader.java new file mode 100644 index 0000000000..8405d26c5d --- /dev/null +++ b/edexOsgi/com.raytheon.uf.edex.archive.feeder/src/com/raytheon/uf/edex/archive/feeder/ArchiveBinaryReader.java @@ -0,0 +1,112 @@ +/** + * This software was developed and / or modified by Raytheon Company, + * pursuant to Contract DG133W-05-CQ-1067 with the US Government. + * + * U.S. EXPORT CONTROLLED TECHNICAL DATA + * This software product contains export-restricted data whose + * export/transfer/disclosure is restricted by U.S. law. Dissemination + * to non-U.S. persons whether in the United States or abroad requires + * an export license or other authorization. + * + * Contractor Name: Raytheon Company + * Contractor Address: 6825 Pine Street, Suite 340 + * Mail Stop B8 + * Omaha, NE 68106 + * 402.291.0100 + * + * See the AWIPS II Master Rights File ("Master Rights File.pdf") for + * further licensing information. + **/ +package com.raytheon.uf.edex.archive.feeder; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; + +import com.raytheon.uf.common.serialization.DynamicSerializationManager; +import com.raytheon.uf.common.serialization.DynamicSerializationManager.SerializationType; +import com.raytheon.uf.common.serialization.SerializationException; + +/** + * This class uses the DynamicSerializationManager to create an instance of the + * first serialized object in a file. + * + * 
+ * 
+ * SOFTWARE HISTORY
+ * 
+ * Date         Ticket#    Engineer    Description
+ * ------------ ---------- ----------- --------------------------
+ * Feb 25, 2014 2838       rferrel     Initial creation
+ * 
+ *
+ * + * @author rferrel + * @version 1.0 + */ + +public class ArchiveBinaryReader { + + /** File containing the serialized object. */ + private File binFile; + + /** Instance of the file's serialized object. */ + Object object; + + /** + * Constructor + * + * @param binFile + * - File containing serialized object. + */ + public ArchiveBinaryReader(File binFile) { + this.binFile = binFile; + } + + /** + * The object obtained from the binFile. + * + * @return object + * @throws FileNotFoundException + * @throws SerializationException + */ + public Object getObject() throws FileNotFoundException, + SerializationException { + if (object == null) { + object = readObject(); + } + return object; + } + + /** + * Create an instance of the file's serialized object. + * + * @return object + * @throws FileNotFoundException + * @throws SerializationException + */ + private Object readObject() throws FileNotFoundException, + SerializationException { + InputStream inputStream = null; + Object object = null; + try { + DynamicSerializationManager dsm = DynamicSerializationManager + .getManager(SerializationType.Thrift); + inputStream = new BufferedInputStream(new FileInputStream(binFile)); + object = dsm.deserialize(inputStream); + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + // Ignore + } + inputStream = null; + } + } + return object; + } +} \ No newline at end of file