xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2499: Update Wes2Bridge Configuration and logging.

Browse source 
Change-Id: I95a530bc8b647c58cff36f449d55bff39943fbc9

Former-commit-id: d76bd3b8ae [formerly 0566c1976e] [formerly 73d0d1236c] [formerly d76bd3b8ae [formerly 0566c1976e] [formerly 73d0d1236c] [formerly 55b226e104 [formerly 73d0d1236c [formerly 9449bc5f562fd638e0a02ea6c2dc4c6828e73561]]]]
Former-commit-id: 55b226e104
Former-commit-id: 324b57d843 [formerly 0a5323a740] [formerly de953d3223b133cc7897e1be8e5c67e47a8bc0aa [formerly 0be00ed984]]
Former-commit-id: 3ea403300e0a5d27621a5036bbace800af6b44f2 [formerly 0317d65b98]
Former-commit-id: 5fb6587980
This commit is contained in:
Richard Peter 2013-11-05 10:07:17 -06:00
parent 6f462ae610
commit 092714717f
24 changed files with 373 additions and 651 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
edexOsgi/build.edex/esb/conf/log4j-ingest.xml
View file

@ -169,7 +169,7 @@
</appender>
<appender name="ThreadBasedLog" class="com.raytheon.uf.edex.log.ThreadBasedAppender">
<param name="ThreadPatterns" value="RadarLog:radarThreadPool.*;SatelliteLog:satelliteThreadPool.*;ShefLog:shefThreadPool.*;TextLog:textThreadPool.*;SmartInitLog:smartInit.*"/>
<param name="ThreadPatterns" value="RadarLog:radarThreadPool.*;SatelliteLog:satelliteThreadPool.*;ShefLog:shefThreadPool.*;TextLog:textThreadPool.*;SmartInitLog:smartInit.*;PurgeLog:Purge.*;ArchiveLog:Archive.*"/>
<param name="DefaultAppender" value="asyncConsole"/>
<appender-ref ref="asyncConsole"/>
<appender-ref ref="RadarLog"/>
@ -177,6 +177,8 @@
<appender-ref ref="ShefLog"/>
<appender-ref ref="SmartInitLog"/>
<appender-ref ref="TextLog"/>
<appender-ref ref="PurgeLog"/>
<appender-ref ref="ArchiveLog"/>
</appender>
<appender name="PerformanceLog" class="org.apache.log4j.rolling.RollingFileAppender">
@ -277,11 +279,6 @@
<level value="INFO"/>
<appender-ref ref="PurgeLog"/>
</logger>
<logger name="com.raytheon.uf.edex.maintenance.archive" additivity="false">
<level value="INFO"/>
<appender-ref ref="ArchiveLog"/>
</logger>
<logger name="RouteFailedLog" additivity="false">
<level value="WARN"/>

6
edexOsgi/com.raytheon.edex.plugin.text/META-INF/MANIFEST.MF
View file

@ -13,11 +13,11 @@ Require-Bundle: com.raytheon.edex.textdb,
com.raytheon.uf.common.serialization.comm,
com.raytheon.uf.edex.decodertools;bundle-version="1.0.0",
com.raytheon.uf.common.status;bundle-version="1.11.17",
com.raytheon.uf.common.site;bundle-version="1.12.1174"
com.raytheon.uf.common.site;bundle-version="1.12.1174",
com.raytheon.uf.edex.archive
Export-Package: com.raytheon.edex.plugin.text,
com.raytheon.edex.plugin.text.dao
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Import-Package: com.raytheon.uf.common.dataplugin.text,
com.raytheon.uf.common.dataplugin.text.db,
com.raytheon.uf.common.dataplugin.text.request,
com.raytheon.uf.edex.maintenance.archive
com.raytheon.uf.common.dataplugin.text.request

7
edexOsgi/com.raytheon.edex.plugin.text/src/com/raytheon/edex/plugin/text/maintenance/archiver/TextArchiveFileNameFormatter.java
View file

@ -34,12 +34,12 @@ import com.raytheon.uf.common.dataplugin.persist.PersistableDataObject;
import com.raytheon.uf.common.dataplugin.text.db.StdTextProduct;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.edex.archive.IPluginArchiveFileNameFormatter;
import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.maintenance.archive.IPluginArchiveFileNameFormatter;
/**
* TODO Add Description
* Properly stores StdTextProducts by time.
*
* <pre>
*
@ -48,7 +48,7 @@ import com.raytheon.uf.edex.maintenance.archive.IPluginArchiveFileNameFormatter;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 20, 2012 dgilling Initial creation
*
* Nov 05, 2013 2499 rjpeter Moved IPluginArchiveFileNameFormatter.
* </pre>
*
* @author dgilling
@ -70,6 +70,7 @@ public class TextArchiveFileNameFormatter implements
* com.raytheon.uf.edex.database.plugin.PluginDao, java.util.Map,
* java.util.Calendar, java.util.Calendar)
*/
@SuppressWarnings("rawtypes")
@Override
public Map<String, List<PersistableDataObject>> getPdosByFile(
String pluginName, PluginDao dao,

7
edexOsgi/com.raytheon.uf.common.archive/src/com/raytheon/uf/common/archive/config/ArchiveConfigManager.java
View file

@ -188,7 +188,7 @@ public class ArchiveConfigManager {
public Collection<ArchiveConfig> getArchives() {
String fileName = ArchiveConstants.selectFileName(Type.Retention, null);
SelectConfig selections = loadSelection(fileName);
if (selections != null && !selections.isEmpty()) {
if ((selections != null) && !selections.isEmpty()) {
try {
for (ArchiveSelect archiveSelect : selections.getArchiveList()) {
ArchiveConfig archiveConfig = archiveMap.get(archiveSelect
@ -407,7 +407,8 @@ public class ArchiveConfigManager {
private Calendar calculateExpiration(ArchiveConfig archive,
CategoryConfig category) {
Calendar expireCal = TimeUtil.newGmtCalendar();
int retHours = category == null || category.getRetentionHours() == 0 ? archive
int retHours = (category == null)
|| (category.getRetentionHours() == 0) ? archive
.getRetentionHours() : category.getRetentionHours();
if (retHours != 0) {
expireCal.add(Calendar.HOUR, (-1) * retHours);
@ -453,7 +454,7 @@ public class ArchiveConfigManager {
for (LocalizationFile lFile : files) {
try {
ArchiveConfig archiveConfig = unmarshalArhiveConfigFromXmlFile(lFile);
if (archiveConfig != null && archiveConfig.isValid()) {
if ((archiveConfig != null) && archiveConfig.isValid()) {
archiveNameToLocalizationFileMap.put(
archiveConfig.getName(), lFile);
archiveMap.put(archiveConfig.getName(), archiveConfig);

23
edexOsgi/com.raytheon.uf.common.dataplugin.text/src/com/raytheon/uf/common/dataplugin/text/db/StdTextProduct.java
View file

@ -58,6 +58,7 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
* 04/06/2010 4734 mhuang Moved from edex server
* 17May2010 2187 cjeanbap Change class to be Abstract
* 27 May 2012 #647 dgilling Implement getIdentifier/setIdentifier.
* Nov 05, 2013 2499 rjpeter Fix generics.
* </pre>
*
* @author jkorman
@ -67,8 +68,8 @@ import com.raytheon.uf.edex.wmo.message.WMOHeader;
@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS)
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public abstract class StdTextProduct extends PersistableDataObject implements
ISerializableObject {
public abstract class StdTextProduct extends
PersistableDataObject<StdTextProductId> implements ISerializableObject {
private static final long serialVersionUID = 1L;
@ -185,10 +186,8 @@ public abstract class StdTextProduct extends PersistableDataObject implements
* (java.lang.Object)
*/
@Override
public void setIdentifier(Object identifier) {
if (identifier instanceof StdTextProductId) {
setProdId((StdTextProductId) identifier);
}
public void setIdentifier(StdTextProductId identifier) {
setProdId(identifier);
}
public String getBbbid() {
@ -227,7 +226,7 @@ public abstract class StdTextProduct extends PersistableDataObject implements
Matcher m = ControlCharacterPattern.matcher(this.product);
String result = this.product;
for (int i = 0; m.find(); ++i) {
for (; m.find();) {
String nonAscii = m.group();
char[] charArr = nonAscii.toCharArray();
if (charArr.length == 1) {
@ -342,10 +341,12 @@ public abstract class StdTextProduct extends PersistableDataObject implements
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((bbbid == null) ? 0 : bbbid.hashCode());
result = prime * result + ((refTime == null) ? 0 : refTime.hashCode());
result = prime * result + ((prodId == null) ? 0 : prodId.hashCode());
result = prime * result + ((product == null) ? 0 : product.hashCode());
result = (prime * result) + ((bbbid == null) ? 0 : bbbid.hashCode());
result = (prime * result)
+ ((refTime == null) ? 0 : refTime.hashCode());
result = (prime * result) + ((prodId == null) ? 0 : prodId.hashCode());
result = (prime * result)
+ ((product == null) ? 0 : product.hashCode());
return result;
}

67
edexOsgi/com.raytheon.uf.common.time/src/com/raytheon/uf/common/time/util/TimeUtil.java
View file

@ -52,6 +52,7 @@ import com.raytheon.uf.common.time.domain.api.ITimePoint;
* Mar 20, 2013 1774 randerso Add SECONDS_PER_DAY, changed SECONDS_PER_HOUR to int.
* Apr 24, 2013 1628 mschenke Added GMT TimeZone Object constant
* Jun 05, 2013 DR 16279 D. Friedman Add timeOfDayToAbsoluteTime
* Nov 05, 2013 2499 rjpeter Added prettyDuration.
* </pre>
*
* @author njensen
@ -165,6 +166,13 @@ public final class TimeUtil {
static final ITimer NULL_CLOCK = new NullClock();
private static final long[] DURATION_INTERVALS = { MILLIS_PER_YEAR,
MILLIS_PER_WEEK, MILLIS_PER_DAY, MILLIS_PER_HOUR,
MILLIS_PER_MINUTE, MILLIS_PER_SECOND };
private static final String[] DURATION_QUALIFIERS = { "y", "w", "d", "h",
"m", "s" };
/**
* The strategy to retrieve the "current time" value from.
*/
@ -415,20 +423,24 @@ public final class TimeUtil {
}
}
/** Converts a time-of-day (in seconds) to an absolute time given an
* absolute reference time. The resulting time is within a day of the
* reference time.
* @param timeOfDaySeconds The time of day in seconds past midnight
* @param referenceTime The reference time (should have GMT time zone)
/**
* Converts a time-of-day (in seconds) to an absolute time given an absolute
* reference time. The resulting time is within a day of the reference time.
*
* @param timeOfDaySeconds
* The time of day in seconds past midnight
* @param referenceTime
* The reference time (should have GMT time zone)
* @return
*/
public static Calendar timeOfDayToAbsoluteTime(int timeOfDaySeconds, Calendar referenceTime) {
public static Calendar timeOfDayToAbsoluteTime(int timeOfDaySeconds,
Calendar referenceTime) {
Calendar targetDay = (Calendar) referenceTime.clone();
int refTimeTodSeconds = referenceTime.get(Calendar.HOUR_OF_DAY) * SECONDS_PER_HOUR
+ referenceTime.get(Calendar.MINUTE) * SECONDS_PER_MINUTE
int refTimeTodSeconds = (referenceTime.get(Calendar.HOUR_OF_DAY) * SECONDS_PER_HOUR)
+ (referenceTime.get(Calendar.MINUTE) * SECONDS_PER_MINUTE)
+ referenceTime.get(Calendar.SECOND);
int absTodDiff = Math.abs(refTimeTodSeconds - timeOfDaySeconds);
if (absTodDiff < SECONDS_PER_DAY - absTodDiff) {
if (absTodDiff < (SECONDS_PER_DAY - absTodDiff)) {
// nothing; use current targetDay
} else if (refTimeTodSeconds < timeOfDaySeconds) {
targetDay.add(Calendar.DAY_OF_MONTH, -1);
@ -442,6 +454,43 @@ public final class TimeUtil {
return targetDay;
}
/**
* Formats millis keeping the two most significant digits.
*
* 1y16w 2d15h 3m5s
*
* @param durationInMillis
* @return
*/
public static String prettyDuration(long durationInMillis) {
StringBuilder timeString = new StringBuilder();
// handle s/ms separately
for (int i = 0; i < (DURATION_INTERVALS.length - 1); i++) {
long interval = DURATION_INTERVALS[i];
if (durationInMillis > interval) {
timeString.append(durationInMillis / interval).append(
DURATION_QUALIFIERS[i]);
durationInMillis %= interval;
timeString.append(durationInMillis / DURATION_INTERVALS[i + 1])
.append(DURATION_QUALIFIERS[i + 1]);
return timeString.toString();
}
}
// seconds/ms
if (durationInMillis > MILLIS_PER_SECOND) {
timeString.append(durationInMillis / MILLIS_PER_SECOND).append('.');
durationInMillis %= MILLIS_PER_SECOND;
int tenth = (int) (durationInMillis / 100);
timeString.append(tenth).append('s');
} else {
timeString.append(durationInMillis).append("ms");
}
return timeString.toString();
}
/**
* Disabled constructor.
*/

12
edexOsgi/com.raytheon.uf.edex.archive/META-INF/MANIFEST.MF
View file

@ -5,13 +5,19 @@ Bundle-SymbolicName: com.raytheon.uf.edex.archive
Bundle-Version: 1.0.0.qualifier
Bundle-Vendor: RAYTHEON
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Export-Package: com.raytheon.uf.edex.archive.purge
Export-Package: com.raytheon.uf.edex.archive,
com.raytheon.uf.edex.archive.purge
Import-Package: com.raytheon.uf.common.archive.config,
com.raytheon.uf.common.archive.request
Require-Bundle: com.raytheon.uf.common.auth;bundle-version="1.12.1174",
com.raytheon.uf.edex.auth;bundle-version="1.12.1174",
com.raytheon.uf.edex.database,
com.raytheon.uf.common.dataplugin,
com.raytheon.uf.common.datastorage,
com.raytheon.uf.common.localization;bundle-version="1.12.1174",
com.raytheon.uf.common.serialization.comm;bundle-version="1.12.1174",
com.raytheon.uf.common.status;bundle-version="1.12.1174",
com.raytheon.uf.common.serialization;bundle-version="1.12.1174",
com.raytheon.uf.common.time,
com.raytheon.uf.common.util;bundle-version="1.12.1174",
com.raytheon.uf.common.localization;bundle-version="1.12.1174"
com.raytheon.uf.edex.auth;bundle-version="1.12.1174",
com.raytheon.uf.edex.core

71
edexOsgi/com.raytheon.uf.edex.archive/res/spring/archive-spring.xml Normal file
View file

@ -0,0 +1,71 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="dataArchiver" class="com.raytheon.uf.edex.archive.DataArchiver">
<constructor-arg value="/archive"/>
</bean>
<bean id="databaseArchiver" class="com.raytheon.uf.edex.archive.DatabaseArchiver"/>
<bean id="databaseArchiverRegistered" factory-bean="dataArchiver" factory-method="registerPluginArchiver" depends-on="dataArchiver">
<constructor-arg ref="databaseArchiver"/>
</bean>
<bean id="archivePurge" class="com.raytheon.uf.edex.archive.purge.ArchivePurger" />
<camelContext id="archive-context"
xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<endpoint id="archiveCron"
uri="clusteredquartz://archive/archiveScheduled/?cron=${archive.cron}"/>
<endpoint id="archivePurgeCron"
uri="clusteredquartz://archive/archivePurgeScheduled/?cron=${archive.purge.cron}" />
<!-- Archive on Scheduled timer -->
<route id="archiveScheduled">
<from uri="archiveCron" />
<doTry>
<to uri="jms-generic:queue:archiveScheduledWork" />
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:archive?level=ERROR" />
</doCatch>
</doTry>
</route>
<route id="archiveScheduledWork">
<from uri="jms-generic:queue:archiveScheduledWork" />
<doTry>
<bean ref="dataArchiver" method="archivePlugins" />
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:archive?level=ERROR" />
</doCatch>
</doTry>
</route>
<!-- Run archivePurge on Scheduled timer -->
<route id="archivePurgeScheduled">
<from uri="archivePurgeCron" />
<to uri="jms-generic:queue:archivePurgeScheduledWork" />
</route>
<route id="archivePurgeScheduledWork">
<from uri="jms-generic:queue:archivePurgeScheduledWork" />
<doTry>
<bean ref="archivePurge" method="purge" />
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:archivePurge?level=ERROR" />
</doCatch>
</doTry>
</route>
</camelContext>
</beans>

33
edexOsgi/com.raytheon.uf.edex.archive/res/spring/archivepurger-spring.xml
View file

@ -1,33 +0,0 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="archivePurge" class="com.raytheon.uf.edex.archive.purge.ArchivePurger" />
<camelContext id="archivePurge-context"
xmlns="http://camel.apache.org/schema/spring" errorHandlerRef="errorHandler">
<endpoint id="archivePurgeCron"
uri="clusteredquartz://archive/archivePurgeScheduled/?cron=${archive.purge.cron}" />
<!-- Run archivePurge on Scheduled timer -->
<route id="archivePurgeScheduled">
<from uri="archivePurgeCron" />
<to uri="jms-generic:queue:archivePurgeScheduledWork" />
</route>
<route id="archivePurgeScheduledWork">
<from uri="jms-generic:queue:archivePurgeScheduledWork" />
<doTry>
<bean ref="archivePurge" method="purge" />
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:archivePurge?level=ERROR" />
</doCatch>
</doTry>
</route>
</camelContext>
</beans>

11
edexOsgi/com.raytheon.uf.edex.archive/resources/com.raytheon.uf.edex.archive.cron.properties
View file

@ -1,6 +1,11 @@
# enable archive
archive.enable=true
# runs database and hdf5 archive for archive server to pull data from
archive.cron=0+40+*+*+*+?
# purge archives
archive.purge.cron=0+5+*+*+*+?
# enable archive purge
archive.purge.enable=false
archive.purge.enable=true
# purge archives
archive.purge.cron=0+5+0/3+*+*+?
# to disable a specific archive, use property archive.disable=pluginName,pluginName...
#archive.disable=grid,text,acars

138
edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DataArchiver.java Normal file
View file

@ -0,0 +1,138 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.archive;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.time.util.ITimer;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
/**
* Handles archiving of data. Has two interfaces for registering data archive.
* Data archived based on archiving for each plugin and general data archive
* programs.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2011 rjpeter Initial creation
* Nov 05, 2013 2499 rjpeter Repackaged, updated to use System properties.
* </pre>
*
* @author rjpeter
* @version 1.0
*/
public class DataArchiver {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(DataArchiver.class);
// enables/disables archiving as a whole
private final static String ENABLE_PROPERTY = "archive.enable";
// allows for disabling of specific plugins if desired
private final static String DISABLE_PROPERTY = "archive.disable";
private final boolean ARCHIVING_ENABLED;
private final Set<String> DISABLED_PLUGINS;
private final List<IPluginArchiver> pluginArchivers = new LinkedList<IPluginArchiver>();
private final List<IDataArchiver> dataArchivers = new LinkedList<IDataArchiver>();
private String archivePath = null;
public DataArchiver(String archivePath) {
this.archivePath = archivePath;
ARCHIVING_ENABLED = Boolean.getBoolean(ENABLE_PROPERTY);
String disabledPluginList = System.getProperty(DISABLE_PROPERTY);
if (disabledPluginList != null) {
String[] plugins = disabledPluginList.split(",");
DISABLED_PLUGINS = new HashSet<String>(plugins.length);
for (String plugin : plugins) {
DISABLED_PLUGINS.add(plugin.trim());
}
} else {
DISABLED_PLUGINS = Collections.emptySet();
}
}
public void archivePlugins() {
Thread.currentThread().setName("Archiver");
if (ARCHIVING_ENABLED) {
ITimer timer = TimeUtil.getTimer();
timer.start();
statusHandler.info("Archival of plugin data started");
// get list of plugins, ordered by plugin
Set<String> availablePlugins = new TreeSet<String>(PluginRegistry
.getInstance().getRegisteredObjects());
for (String pluginName : availablePlugins) {
if (DISABLED_PLUGINS.contains(pluginName)) {
statusHandler.info(pluginName + ": Archiving disabled");
} else {
for (IPluginArchiver pluginArchiver : pluginArchivers) {
pluginArchiver.archivePlugin(pluginName, archivePath);
}
}
}
timer.stop();
statusHandler
.info("Archival of plugin data completed. Time to run: "
+ TimeUtil.prettyDuration(timer.getElapsedTime()));
} else {
statusHandler.info("Archival of plugin data disabled, exiting");
}
}
public Object registerPluginArchiver(IPluginArchiver archiver) {
if (!pluginArchivers.contains(archiver)) {
pluginArchivers.add(archiver);
} else {
statusHandler.warn("Plugin archiver already registered: "
+ archiver);
}
return this;
}
public Object registerDataArchiver(IDataArchiver archiver) {
if (!dataArchivers.contains(archiver)) {
dataArchivers.add(archiver);
} else {
statusHandler.warn("Data archiver already registered: " + archiver);
}
return this;
}
}

77
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/DatabaseArchiver.java → edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DatabaseArchiver.java
View file

@ -17,10 +17,8 @@
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
package com.raytheon.uf.edex.archive;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -55,6 +53,7 @@ import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.TimeUtil;
import com.raytheon.uf.common.util.FileUtil;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.database.DataAccessLayerException;
@ -64,7 +63,6 @@ import com.raytheon.uf.edex.database.cluster.ClusterTask;
import com.raytheon.uf.edex.database.cluster.handler.CurrentTimeClusterLockHandler;
import com.raytheon.uf.edex.database.plugin.PluginDao;
import com.raytheon.uf.edex.database.plugin.PluginFactory;
import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
/**
* This class handles moving processed data to the archiver directory.
@ -78,7 +76,7 @@ import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
* Nov 17, 2011 rjpeter Initial creation
* Jan 18, 2013 1469 bkowal Removed the hdf5 data directory.
* Oct 23, 2013 2478 rferrel Make date format thread safe.
*
* Nov 05, 2013 2499 rjpeter Repackaged, removed config files, always compresses.
* </pre>
*
* @author rjpeter
@ -125,8 +123,7 @@ public class DatabaseArchiver implements IPluginArchiver {
}
@Override
public void archivePlugin(String pluginName, String archivePath,
DataArchiveConfig conf) {
public void archivePlugin(String pluginName, String archivePath) {
PluginProperties props = PluginRegistry.getInstance()
.getRegisteredObject(pluginName);
if ((props != null) && (props.getRecord() != null)
@ -135,7 +132,7 @@ public class DatabaseArchiver implements IPluginArchiver {
if (recordClass != null) {
try {
recordClass.asSubclass(PluginDataObject.class);
archivePluginData(pluginName, archivePath, conf);
archivePluginData(pluginName, archivePath);
} catch (ClassCastException e) {
// not an error, using asSubClass to filter non
// PluginDataObjects
@ -145,8 +142,7 @@ public class DatabaseArchiver implements IPluginArchiver {
}
@SuppressWarnings("rawtypes")
public boolean archivePluginData(String pluginName, String archivePath,
DataArchiveConfig conf) {
public boolean archivePluginData(String pluginName, String archivePath) {
SimpleDateFormat dateFormat = TL_DATE_FORMAT.get();
// set archive time
Calendar runTime = Calendar.getInstance();
@ -184,7 +180,7 @@ public class DatabaseArchiver implements IPluginArchiver {
Set<String> datastoreFilesToArchive = new HashSet<String>();
startTime = determineStartTime(pluginName, ct.getExtraInfo(),
runTime, dao, conf);
runTime, dao);
Calendar endTime = determineEndTime(startTime, runTime);
Map<String, List<PersistableDataObject>> pdoMap = new HashMap<String, List<PersistableDataObject>>();
@ -201,7 +197,7 @@ public class DatabaseArchiver implements IPluginArchiver {
if ((pdosToSave != null) && !pdosToSave.isEmpty()) {
recordCount += savePdoMap(pluginName, archivePath,
pdosToSave, conf.getCompressionEnabled());
pdosToSave);
for (Map.Entry<String, List<PersistableDataObject>> entry : pdosToSave
.entrySet()) {
List<PersistableDataObject> pdoList = entry.getValue();
@ -217,8 +213,7 @@ public class DatabaseArchiver implements IPluginArchiver {
}
if ((pdoMap != null) && !pdoMap.isEmpty()) {
recordCount += savePdoMap(pluginName, archivePath, pdoMap,
conf.getCompressionEnabled());
recordCount += savePdoMap(pluginName, archivePath, pdoMap);
// don't forget to archive the HDF5 for the records that weren't
// saved off by the prior while block
for (Map.Entry<String, List<PersistableDataObject>> entry : pdoMap
@ -257,15 +252,11 @@ public class DatabaseArchiver implements IPluginArchiver {
try {
// data must be older than 30 minutes, and no older than
// hours
// to keep hours need to lookup plugin and see if
// compression
// matches, or embed in configuration the compression
// level on
// archive, but would still need to lookup plugin
ds.copy(outputDir, compRequired, "lastArchived",
1800000,
conf.getHoursToKeep() * 60000 + 1800000);
// hours to keep hours need to lookup plugin and see if
// compression matches, or embed in configuration the
// compression level on archive, but would still need to
// lookup plugin
ds.copy(outputDir, compRequired, "lastArchived", 0, 0);
} catch (StorageException e) {
statusHandler.handle(Priority.PROBLEM,
e.getLocalizedMessage());
@ -280,10 +271,12 @@ public class DatabaseArchiver implements IPluginArchiver {
}
if (recordCount > 0) {
statusHandler.info(pluginName + ": successfully archived "
+ recordCount + " records in "
+ (System.currentTimeMillis() - timimgStartMillis)
+ " ms");
statusHandler.info(pluginName
+ ": successfully archived "
+ recordCount
+ " records in "
+ TimeUtil.prettyDuration(System.currentTimeMillis()
- timimgStartMillis));
} else {
statusHandler
.info(pluginName + ": Found no records to archive");
@ -309,9 +302,8 @@ public class DatabaseArchiver implements IPluginArchiver {
@SuppressWarnings("rawtypes")
protected int savePdoMap(String pluginName, String archivePath,
Map<String, List<PersistableDataObject>> pdoMap,
boolean compressMetadata) throws SerializationException,
IOException {
Map<String, List<PersistableDataObject>> pdoMap)
throws SerializationException, IOException {
int recordsSaved = 0;
for (Map.Entry<String, List<PersistableDataObject>> entry : pdoMap
@ -324,7 +316,7 @@ public class DatabaseArchiver implements IPluginArchiver {
path = path.substring(0, path.length() - 3);
}
path += (compressMetadata ? ".bin.gz" : ".bin");
path += ".bin.gz";
File file = new File(path);
List<PersistableDataObject> pdosToSerialize = entry.getValue();
@ -337,10 +329,7 @@ public class DatabaseArchiver implements IPluginArchiver {
try {
// created gzip'd stream
is = (compressMetadata ? new GZIPInputStream(
new FileInputStream(file), 8192)
: new BufferedInputStream(
new FileInputStream(file), 8192));
is = new GZIPInputStream(new FileInputStream(file), 8192);
// transform back for list append
@SuppressWarnings("unchecked")
@ -397,10 +386,7 @@ public class DatabaseArchiver implements IPluginArchiver {
}
// created gzip'd stream
os = (compressMetadata ? new GZIPOutputStream(
new FileOutputStream(file), 8192)
: new BufferedOutputStream(new FileOutputStream(file),
8192));
os = new GZIPOutputStream(new FileOutputStream(file), 8192);
// Thrift serialize pdo list
SerializationUtil.transformToThriftUsingStream(pdosToSerialize,
@ -427,13 +413,11 @@ public class DatabaseArchiver implements IPluginArchiver {
* @param extraInfo
* @param runTime
* @param dao
* @param conf
* @return startTime
* @throws DataAccessLayerException
*/
protected Calendar determineStartTime(String pluginName, String extraInfo,
Calendar runTime, PluginDao dao, DataArchiveConfig conf)
throws DataAccessLayerException {
Calendar runTime, PluginDao dao) throws DataAccessLayerException {
Calendar startTime = null;
SimpleDateFormat dateFormat = TL_DATE_FORMAT.get();
@ -474,14 +458,7 @@ public class DatabaseArchiver implements IPluginArchiver {
}
}
// earliest time based on default retention
Calendar earliestTime = Calendar.getInstance(TimeZone
.getTimeZone("GMT"));
earliestTime
.add(Calendar.HOUR, (-1 * conf.getHoursToKeep().intValue()));
return (startTime.compareTo(earliestTime) < 0) ? earliestTime
: startTime;
return startTime;
}
/**

4
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/DefaultPluginArchiveFileNameFormatter.java → edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/DefaultPluginArchiveFileNameFormatter.java
View file

@ -17,7 +17,7 @@
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
package com.raytheon.uf.edex.archive;
import java.io.File;
import java.util.Calendar;
@ -51,7 +51,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Mar 12, 2013 1783 rferrel Replace ArrayList with LinkedList to
* remove excess capacity and reduce
* time to resize a growing list.
*
* Nov 05, 2013 2499 rjpeter Repackaged
* </pre>
*
* @author dgilling

6
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/IDataArchiver.java → edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/IDataArchiver.java
View file

@ -17,10 +17,10 @@
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
package com.raytheon.uf.edex.archive;
/**
* TODO Add Description
* Data Archiver interface
*
* <pre>
*
@ -29,7 +29,7 @@ package com.raytheon.uf.edex.maintenance.archive;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2011 rjpeter Initial creation
*
* Nov 05, 2013 2499 rjpeter Repackaged
* </pre>
*
* @author rjpeter

7
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/IPluginArchiveFileNameFormatter.java → edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/IPluginArchiveFileNameFormatter.java
View file

@ -17,7 +17,7 @@
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
package com.raytheon.uf.edex.archive;
import java.util.Calendar;
import java.util.List;
@ -28,7 +28,7 @@ import com.raytheon.uf.edex.database.DataAccessLayerException;
import com.raytheon.uf.edex.database.plugin.PluginDao;
/**
* TODO Add Description
* Interface for archive file name formatters.
*
* <pre>
*
@ -37,7 +37,7 @@ import com.raytheon.uf.edex.database.plugin.PluginDao;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 20, 2012 dgilling Initial creation
*
* Nov 05, 2013 2499 rjpeter Repackaged
* </pre>
*
* @author dgilling
@ -64,6 +64,7 @@ public interface IPluginArchiveFileNameFormatter {
* If the DAO is unable to retrieve the records from the
* database.
*/
@SuppressWarnings("rawtypes")
public abstract Map<String, List<PersistableDataObject>> getPdosByFile(
String pluginName, PluginDao dao,
Map<String, List<PersistableDataObject>> pdoMap,

9
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/IPluginArchiver.java → edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/IPluginArchiver.java
View file

@ -17,9 +17,7 @@
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
package com.raytheon.uf.edex.archive;
/**
* Interface for archiving data based on plugins.
@ -31,7 +29,7 @@ import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2011 rjpeter Initial creation
*
* Nov 05, 2013 2499 rjpeter Repackaged
* </pre>
*
* @author rjpeter
@ -39,6 +37,5 @@ import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
*/
public interface IPluginArchiver {
public void archivePlugin(String pluginName, String archivePath,
DataArchiveConfig config);
public void archivePlugin(String pluginName, String archivePath);
}

21
edexOsgi/com.raytheon.uf.edex.archive/src/com/raytheon/uf/edex/archive/purge/ArchivePurger.java
View file

@ -26,6 +26,8 @@ import com.raytheon.uf.common.archive.config.ArchiveConfigManager;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.common.time.util.ITimer;
import com.raytheon.uf.common.time.util.TimeUtil;
/**
* Purge task to purge archived data based on configured expiration.
@ -41,7 +43,7 @@ import com.raytheon.uf.common.status.UFStatus.Priority;
* Aug 28, 2013 2299 rferrel manager.purgeExpiredFromArchive now returns
* number of files purged.
* Sep 03, 2013 2224 rferrel Add check to enable/disable purger.
*
* Nov 05, 2013 2499 rjpeter Repackaged
* </pre>
*
* @author bgonzale
@ -58,12 +60,17 @@ public class ArchivePurger {
* Purge expired elements from the archives.
*/
public static void purge() {
Thread.currentThread().setName("Purge-Archive");
String enableString = System.getProperty(ENABLE_PROPERTY, "false");
if (Boolean.parseBoolean(enableString)) {
statusHandler.info("::Archive Purged started.");
ITimer timer = TimeUtil.getTimer();
timer.start();
statusHandler.info("Archive Purge started.");
ArchiveConfigManager manager = ArchiveConfigManager.getInstance();
Collection<ArchiveConfig> archives = manager.getArchives();
for (ArchiveConfig archive : archives) {
ITimer archiveTimer = TimeUtil.getTimer();
archiveTimer.start();
int purgeCount = manager.purgeExpiredFromArchive(archive);
if (statusHandler.isPriorityEnabled(Priority.INFO)) {
StringBuilder sb = new StringBuilder(archive.getName());
@ -73,11 +80,17 @@ public class ArchivePurger {
if (purgeCount != 1) {
sb.append("s");
}
sb.append(".");
sb.append(" in ")
.append(TimeUtil.prettyDuration(archiveTimer
.getElapsedTime())).append(".");
statusHandler.info(sb.toString());
}
}
statusHandler.info("::Archive Purged finished.");
statusHandler.info("Archive Purge finished. Time to run: "
+ TimeUtil.prettyDuration(timer.getElapsedTime()));
} else {
statusHandler.info("Archive Purge disabled, exiting");
}
}
}

19
edexOsgi/com.raytheon.uf.edex.maintenance/META-INF/MANIFEST.MF
View file

@ -5,22 +5,11 @@ Bundle-SymbolicName: com.raytheon.uf.edex.maintenance
Bundle-Version: 1.0.0.qualifier
Bundle-Vendor: RAYTHEON
Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Import-Package: com.raytheon.uf.common.dataplugin,
com.raytheon.uf.common.dataplugin.persist,
com.raytheon.uf.common.dataquery.db,
Require-Bundle: com.raytheon.uf.common.dataplugin,
com.raytheon.uf.common.datastorage,
com.raytheon.uf.common.localization,
com.raytheon.uf.common.serialization,
com.raytheon.uf.common.status,
com.raytheon.uf.common.time,
com.raytheon.uf.common.util,
com.raytheon.uf.common.util.registry,
com.raytheon.uf.edex.core.dataplugin,
com.raytheon.uf.edex.core.props,
com.raytheon.uf.edex.database,
com.raytheon.uf.edex.database.cluster,
com.raytheon.uf.edex.database.cluster.handler,
com.raytheon.uf.edex.database.plugin,
com.raytheon.uf.edex.pointdata,
org.springframework.orm.hibernate3.support
Export-Package: com.raytheon.uf.edex.maintenance.archive
com.raytheon.uf.edex.core,
com.raytheon.uf.edex.pointdata

1
edexOsgi/com.raytheon.uf.edex.maintenance/META-INF/services/com.raytheon.uf.common.serialization.ISerializableObject
View file

@ -1 +0,0 @@
com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig

8
edexOsgi/com.raytheon.uf.edex.maintenance/component-deploy.xml
View file

@ -1,8 +0,0 @@
<project basedir="." default="deploy" name="com.raytheon.uf.edex.maintenance">
<available file="../build.edex" property="build.dir.location" value="../build.edex"/>
<available file="../../../../../build.edex" property="build.dir.location" value="../../../../../build.edex"/>
<import file="${build.dir.location}/basebuilds/component_deploy_base.xml" />
</project>

46
edexOsgi/com.raytheon.uf.edex.maintenance/res/spring/maintenance-ingest.xml
View file

@ -8,32 +8,10 @@
<constructor-arg value="LZF" />
</bean>
<bean id="dataArchiver" class="com.raytheon.uf.edex.maintenance.archive.DataArchiver">
<constructor-arg value="/archive"/>
</bean>
<bean id="databaseArchiver" class="com.raytheon.uf.edex.maintenance.archive.DatabaseArchiver"/>
<camelContext id="maintenanceContext" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler">
<bean id="dataStoreArchiver" class="com.raytheon.uf.edex.maintenance.archive.DataStoreArchiver">
<!-- the compression to archive at, valid values are NONE or LZF -->
<constructor-arg value="LZF" />
</bean>
<bean id="databaseArchiverRegistered" factory-bean="dataArchiver" factory-method="registerPluginArchiver" depends-on="dataArchiver">
<constructor-arg ref="databaseArchiver"/>
</bean>
<!-- Need to register with databaseArchiver for archiving associated data store
<bean id="datastoreArchiverRegistered" factory-bean="dataArchiver" factory-method="registerPluginArchiver">
<constructor-arg ref="dataStoreArchiver"/>
</bean>
-->
<camelContext id="clusteredMaintenanceContext" xmlns="http://camel.apache.org/schema/spring"
errorHandlerRef="errorHandler" autoStartup="false">
<endpoint id="repackCron" uri="quartz://repack/repackScheduled/?cron=${repack.cron}"/>
<endpoint id="archiveCron" uri="quartz://archive/archiveScheduled/?cron=${archive.cron}"/>
<endpoint id="repackCron" uri="clusteredquartz://repack/repackScheduled/?cron=${repack.cron}"/>
<!-- Repack on Scheduled timer -->
<route id="repackScheduled">
@ -47,23 +25,5 @@
</doCatch>
</doTry>
</route>
<!-- Archive on Scheduled timer -->
<route id="archiveScheduled">
<from uri="archiveCron" />
<doTry>
<bean ref="dataArchiver" method="archivePlugins" />
<doCatch>
<exception>java.lang.Throwable</exception>
<to
uri="log:archive?level=ERROR" />
</doCatch>
</doTry>
</route>
</camelContext>
<bean factory-bean="clusteredCamelContextMgr"
factory-method="register">
<constructor-arg ref="clusteredMaintenanceContext" />
</bean>
</beans>

232
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/DataArchiver.java
View file

@ -1,232 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationType;
import com.raytheon.uf.common.localization.LocalizationFile;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.serialization.SerializationUtil;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
/**
* Handles archiving of data. Has two interfaces for registering data archive.
* Data archived based on archiving for each plugin and general data archive
* programs.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 16, 2011 rjpeter Initial creation
*
* </pre>
*
* @author rjpeter
* @version 1.0
*/
public class DataArchiver {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(DataArchiver.class);
private List<IPluginArchiver> pluginArchivers = new ArrayList<IPluginArchiver>();
private List<IDataArchiver> dataArchivers = new ArrayList<IDataArchiver>();
private String archivePath = null;
private String defaultPlugin = "default";
private String configDir = "archiver";
public DataArchiver(String archivePath) {
this.archivePath = archivePath;
}
public void archivePlugins() {
statusHandler.info("Archival of plugin data starting");
// get list of plugins, ordered by plugin
Set<String> availablePlugins = new TreeSet<String>(PluginRegistry
.getInstance().getRegisteredObjects());
Map<String, DataArchiveConfig> configs = getDataArchiveConfigs();
DataArchiveConfig defaultConf = configs.get(defaultPlugin);
File baseArchive = new File(archivePath);
for (String pluginName : availablePlugins) {
DataArchiveConfig conf = configs.get(pluginName);
if (conf == null) {
conf = defaultConf;
}
if (Boolean.TRUE.equals(conf.getArchivingEnabled())) {
for (IPluginArchiver pluginArchiver : pluginArchivers) {
pluginArchiver.archivePlugin(pluginName, archivePath, conf);
}
}
}
statusHandler.info("Archival of plugin data complete");
}
public Object registerPluginArchiver(IPluginArchiver archiver) {
if (!pluginArchivers.contains(archiver)) {
pluginArchivers.add(archiver);
} else {
statusHandler.warn("Plugin archiver already registered: "
+ archiver);
}
return this;
}
public Object registerDataArchiver(IDataArchiver archiver) {
if (!dataArchivers.contains(archiver)) {
dataArchivers.add(archiver);
} else {
statusHandler.warn("Data archiver already registered: " + archiver);
}
return this;
}
private Map<String, DataArchiveConfig> getDataArchiveConfigs() {
Map<String, DataArchiveConfig> configs = new HashMap<String, DataArchiveConfig>();
IPathManager pathMgr = PathManagerFactory.getPathManager();
// process in reverse order so BASE is processed before CONFIGURED
// before SITE
List<LocalizationContext> contexts = Arrays.asList(pathMgr
.getLocalSearchHierarchy(LocalizationType.COMMON_STATIC));
Collections.reverse(contexts);
String[] extensions = new String[] { "xml" };
for (LocalizationContext ctx : contexts) {
statusHandler.info("Loading context: " + ctx);
LocalizationFile[] lfs = pathMgr.listFiles(ctx, configDir,
extensions, false, true);
if (lfs != null && lfs.length > 0) {
for (LocalizationFile lf : lfs) {
String fileName = lf.getName();
try {
File f = lf.getFile(true);
fileName = f.getAbsolutePath();
Object obj = SerializationUtil
.jaxbUnmarshalFromXmlFile(f);
if (obj instanceof DataArchiveConfig) {
DataArchiveConfig conf = (DataArchiveConfig) obj;
String plugin = conf.getPluginName();
if (plugin != null) {
plugin = plugin.trim();
if (!plugin.isEmpty()) {
configs.put(plugin, conf);
} else {
throw new Exception(
"Configuration file does not specify pluginName");
}
} else {
throw new Exception(
"Configuration file does not specify pluginName");
}
} else {
throw new Exception(
"File in wrong format, expected "
+ DataArchiveConfig.class
+ ", found " + obj.getClass());
}
} catch (Throwable e) {
statusHandler.error(
"Failed to load archive configuration file: "
+ fileName, e);
}
}
}
}
DataArchiveConfig defaultConf = configs.get(defaultPlugin);
if (defaultConf == null) {
// default plugin didn't load from disk, force a default config
statusHandler
.warn("Failed to find default configuration, using internal defaults");
defaultConf = new DataArchiveConfig();
defaultConf.setPluginName(defaultPlugin);
configs.put(defaultPlugin, defaultConf);
}
if (!defaultConf.isArchivingEnabledSet()) {
defaultConf.setArchivingEnabled(Boolean.TRUE);
}
if (!defaultConf.isCompressionEnabledSet()) {
defaultConf.setCompressionEnabled(Boolean.TRUE);
}
if (!defaultConf.isHoursToKeepSet()) {
defaultConf.setHoursToKeep(6);
}
// override unset fields with default
for (DataArchiveConfig pluginConf : configs.values()) {
if (pluginConf.getPluginName().equals(defaultPlugin)) {
// skip default conf
continue;
}
if (!pluginConf.isArchivingEnabledSet()) {
pluginConf.setArchivingEnabled(defaultConf
.getArchivingEnabled());
}
if (!pluginConf.isCompressionEnabledSet()) {
pluginConf.setCompressionEnabled(defaultConf
.getArchivingEnabled());
}
if (!pluginConf.isHoursToKeepSet()) {
pluginConf.setHoursToKeep(defaultConf.getHoursToKeep());
}
}
try {
statusHandler.info("DefaultConfiguration:\n"
+ SerializationUtil.marshalToXml(defaultConf));
} catch (Exception e) {
statusHandler.handle(Priority.WARN, "Failed to deserialize config",
e);
}
return configs;
}
}

79
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/DataStoreArchiver.java
View file

@ -1,79 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive;
import java.io.File;
import com.raytheon.uf.common.datastorage.DataStoreFactory;
import com.raytheon.uf.common.datastorage.IDataStore;
import com.raytheon.uf.common.datastorage.StorageException;
import com.raytheon.uf.common.datastorage.StorageProperties.Compression;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
import com.raytheon.uf.edex.maintenance.archive.config.DataArchiveConfig;
/**
* Uses the repack feature of IDataStore to archive data by repacking it to a
* specified compression at the hdf5 dataset level and moving the resulting file
* to the archive dir.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Dec 8, 2011 njensen Initial creation
* Jan 14, 2013 1469 bkowal Removed the hdf5 data directory.
* Jul 23, 2013 2216 rferrel Removed the time stamp filter in hdf5 copy.
*
* </pre>
*
* @author njensen
* @version 1.0
*/
public class DataStoreArchiver {
private static final transient IUFStatusHandler statusHandler = UFStatus
.getHandler(DataStoreArchiver.class);
private Compression compression = Compression.NONE;
public DataStoreArchiver(String compression) {
this.compression = Compression.valueOf(compression);
}
public void archiveFiles(String[] hdf5Files, String archiveDir,
DataArchiveConfig conf) {
for (String hdf5File : hdf5Files) {
IDataStore ds = DataStoreFactory.getDataStore(new File(hdf5File));
String outputDir = archiveDir; // + dirs of hdf5 file
try {
// Do not perform time stamp check.
ds.copy(outputDir, compression, null, 0, 0);
} catch (StorageException e) {
statusHandler.handle(Priority.PROBLEM, e.getLocalizedMessage());
}
}
}
}

131
edexOsgi/com.raytheon.uf.edex.maintenance/src/com/raytheon/uf/edex/maintenance/archive/config/DataArchiveConfig.java
View file

@ -1,131 +0,0 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.edex.maintenance.archive.config;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
/**
* Data archive configuration. Configuration should be pulled from common_static
* localization. Configuration with a pluginName of default will all to all
* plugins.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Jan 14, 2012 rjpeter Initial creation
*
* </pre>
*
* @author rjpeter
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
public class DataArchiveConfig {
@XmlElement
private String pluginName;
@XmlElement
private Integer hoursToKeep;
@XmlElement
private Boolean archivingEnabled;
@XmlElement
private Boolean compressionEnabled;
/**
* @return the pluginName
*/
public String getPluginName() {
return pluginName;
}
/**
* @param pluginName
* the pluginName to set
*/
public void setPluginName(String pluginName) {
this.pluginName = pluginName;
}
/**
* @return the hoursToKeep
*/
public Integer getHoursToKeep() {
return hoursToKeep;
}
/**
* @param hoursToKeep
* the hoursToKeep to set
*/
public void setHoursToKeep(Integer hoursToKeep) {
this.hoursToKeep = hoursToKeep;
}
/**
* @return the archivingEnabled
*/
public Boolean getArchivingEnabled() {
return archivingEnabled;
}
/**
* @param archivingEnabled
* the archivingEnabled to set
*/
public void setArchivingEnabled(Boolean archivingEnabled) {
this.archivingEnabled = archivingEnabled;
}
/**
* @param compressionEnabled
* the compressionEnabled to set
*/
public void setCompressionEnabled(Boolean compressionEnabled) {
this.compressionEnabled = compressionEnabled;
}
/**
* @return the compressionEnabled
*/
public Boolean getCompressionEnabled() {
return compressionEnabled;
}
public boolean isArchivingEnabledSet() {
return archivingEnabled != null;
}
public boolean isHoursToKeepSet() {
return hoursToKeep != null;
}
public boolean isCompressionEnabledSet() {
return (compressionEnabled != null);
}
}