xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #1959 MADIS implementation peice in AWIPS II baseline" into development

Browse source 
Former-commit-id: 49801bd5e2 [formerly 48d8bf941a] [formerly 00dc4787f6] [formerly 8a29b450c4 [formerly 00dc4787f6 [formerly e892ed6958df9141768b9593dee93492d8e707d3]]]
Former-commit-id: 8a29b450c4
Former-commit-id: 8aca75081b62bccf75287ba01aa868a459ab98c3 [formerly f00d9146e4]
Former-commit-id: 6ab64bbbcd
This commit is contained in:
Dustin Johnson 2013-05-02 12:54:37 -05:00 • committed by Gerrit Code Review
commit 1485c27bc1
15 changed files with 587 additions and 209 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
edexOsgi/build.edex/esb/conf/modes.xml
View file

@ -329,15 +329,23 @@
<!-- OGC/DPA services --> <!-- OGC/DPA services -->
<include>ogc-common.xml</include> <include>ogc-common.xml</include>
<include>.*-ogc-request.xml</include> <include>.*-ogc-request.xml</include>
<!-- Purge OGC/DPA registred plugins -->
<include>purge-spring.xml</include>
<include>ogc-purge.xml</include>
</mode> </mode>
<!-- This is a sample reference implmentation --> <!-- This is MADIS implmentation of dataprovideragent -->
<mode name="dataprovideragent"> <mode name="dataprovideragent">
<includeMode>dataProviderAgentTemplate</includeMode> <includeMode>dataProviderAgentTemplate</includeMode>
<!-- pointdata/obs specific services --> <!-- pointdata/madis specific services -->
<include>madis-common.xml</include>
<include>pointdata-common.xml</include>
<include>madis-dpa-ingest.xml</include>
<include>madis-ogc.xml</include>-->
<!-- pointdata/obs specific services
<include>obs-common.xml</include> <include>obs-common.xml</include>
<include>pointdata-common.xml</include> <include>pointdata-common.xml</include>
<include>obs-dpa-ingest.xml</include> <include>obs-dpa-ingest.xml</include>
<include>obs-ogc.xml</include> <include>obs-ogc.xml</include>-->
<!-- grid specific services <!-- grid specific services
<include>grib-common.xml</include> <include>grib-common.xml</include>
<include>grid-staticdata-process.xml</include> <include>grid-staticdata-process.xml</include>

11
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/ConfigLayer.java
View file

@ -65,6 +65,9 @@ public class ConfigLayer implements ISerializableObject {
@XmlElement(name = "maxy", required = true) @XmlElement(name = "maxy", required = true)
private Double maxy; private Double maxy;
@XmlElement(name = "crs", required = true)
private String crs;
@XmlElements({ @XmlElement(name = "parameter", type = Parameter.class, required = true) }) @XmlElements({ @XmlElement(name = "parameter", type = Parameter.class, required = true) })
private List<Parameter> parameters; private List<Parameter> parameters;
@ -127,4 +130,12 @@ public class ConfigLayer implements ISerializableObject {
} }
return parm; return parm;
} }
public void setCrs(String crs) {
this.crs = crs;
}
public String getCrs() {
return crs;
}
} }

37
edexOsgi/com.raytheon.uf.common.datadelivery.harvester/src/com/raytheon/uf/common/datadelivery/harvester/HarvesterConfig.java
View file

@ -18,6 +18,7 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* 12 Sept, 2012 1038 dhladky Initial creation * 12 Sept, 2012 1038 dhladky Initial creation
* 1 May 2013 1959 dhladky remove backup registry references
* *
* </pre> * </pre>
* *
@ -33,18 +34,6 @@ public class HarvesterConfig implements ISerializableObject {
@DynamicSerializeElement @DynamicSerializeElement
private Provider provider; private Provider provider;
@XmlElement(name = "primaryRegistryHost")
@DynamicSerializeElement
private String primaryRegistryHost;
@XmlElement(name = "secondaryRegistryHost")
@DynamicSerializeElement
private String secondaryRegistryHost;
@XmlElement(name = "tertiaryRegistryHost")
@DynamicSerializeElement
private String tertiaryRegistryHost;
@XmlElement(name = "agent") @XmlElement(name = "agent")
@DynamicSerializeElement @DynamicSerializeElement
private Agent agent; private Agent agent;
@ -57,40 +46,16 @@ public class HarvesterConfig implements ISerializableObject {
return agent; return agent;
} }
public String getPrimaryRegistryHost() {
return primaryRegistryHost;
}
public Provider getProvider() { public Provider getProvider() {
return provider; return provider;
} }
public String getSecondaryRegistryHost() {
return secondaryRegistryHost;
}
public String getTertiaryRegistryHost() {
return tertiaryRegistryHost;
}
public void setAgent(Agent agent) { public void setAgent(Agent agent) {
this.agent = agent; this.agent = agent;
} }
public void setPrimaryRegistryHost(String primaryRegistryHost) {
this.primaryRegistryHost = primaryRegistryHost;
}
public void setProvider(Provider provider) { public void setProvider(Provider provider) {
this.provider = provider; this.provider = provider;
} }
public void setSecondaryRegistryHost(String secondaryRegistryHost) {
this.secondaryRegistryHost = secondaryRegistryHost;
}
public void setTertiaryRegistryHost(String tertiaryRegistryHost) {
this.tertiaryRegistryHost = tertiaryRegistryHost;
}
} }

3
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/META-INF/MANIFEST.MF
View file

@ -32,7 +32,8 @@ Require-Bundle: org.eclipse.core.runtime,
com.raytheon.uf.edex.registry.ebxml;bundle-version="1.0.0", com.raytheon.uf.edex.registry.ebxml;bundle-version="1.0.0",
com.raytheon.uf.common.registry.ebxml;bundle-version="1.0.0", com.raytheon.uf.common.registry.ebxml;bundle-version="1.0.0",
com.raytheon.uf.common.stats;bundle-version="1.0.0", com.raytheon.uf.common.stats;bundle-version="1.0.0",
com.raytheon.uf.common.datadelivery.harvester;bundle-version="1.0.0" com.raytheon.uf.common.datadelivery.harvester;bundle-version="1.0.0",
com.raytheon.uf.edex.purgesrv;bundle-version="1.12.1174"
Bundle-RequiredExecutionEnvironment: JavaSE-1.6 Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Bundle-ActivationPolicy: lazy Bundle-ActivationPolicy: lazy
Export-Package: com.raytheon.uf.edex.datadelivery.harvester;uses:="com.raytheon.uf.edex.datadelivery.harvester.crawler", Export-Package: com.raytheon.uf.edex.datadelivery.harvester;uses:="com.raytheon.uf.edex.datadelivery.harvester.crawler",

14
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/res/spring/ogc-purge.xml Normal file
View file

@ -0,0 +1,14 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
<bean id="purgeManager" class="com.raytheon.uf.edex.datadelivery.harvester.purge.OGCPurgeManager">
<property name="clusterLimit" value="${purge.clusterlimit}"/>
<property name="serverLimit" value="${purge.serverlimit}"/>
<property name="deadPurgeJobAge" value="${purge.deadjobage}"/>
<property name="purgeFrequency" value="${purge.frequency}"/>
<property name="fatalFailureCount" value="${purge.fatalfailurecount}"/>
<property name="purgeEnabled" value="${purge.enabled}"/>
</bean>
</beans>

67
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/src/com/raytheon/uf/edex/datadelivery/harvester/purge/OGCPurgeManager.java Normal file
View file

@ -0,0 +1,67 @@
package com.raytheon.uf.edex.datadelivery.harvester.purge;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
import java.util.ArrayList;
import java.util.List;
import com.raytheon.uf.edex.core.dataplugin.PluginRegistry;
import com.raytheon.uf.edex.purgesrv.PurgeManager;
/*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Apr 11, 2013 #1959 dhladky Our own OGC purger
*
* </pre>
*
* @author dhladky
* @version 1.0
*/
public class OGCPurgeManager extends PurgeManager {
/**
* Creates a new PurgeManager
*/
protected OGCPurgeManager() {
}
/**
* Executes the purge only on available plugins that are registered with
* camel. This works better for the DPA instances that will be running it.
* They aren't required to purge data from plugins that aren't registerd to
* their JVM which would be highly wasteful in this instance.
*/
public void executePurge() {
// check for any new plugins or database being purged and needing
// entries recreated
List<String> availablePlugins = new ArrayList<String>(PluginRegistry
.getInstance().getRegisteredObjects());
purgeRunner(availablePlugins);
}
}

293
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/MADIS-harvester.xml Normal file
View file

@ -0,0 +1,293 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<harvester xmlns:ns2="com.raytheon.uf.common.datadelivery.registry"
xmlns:ns4="http://www.w3.org/1999/xlink" xmlns:ns3="urn:oasis:names:tc:ebxml-regrep:xsd:rim:4.0"
xmlns:ns9="urn:oasis:names:tc:ebxml-regrep:xsd:lcm:4.0" xmlns:ns5="http://www.w3.org/2005/08/addressing"
xmlns:ns6="urn:oasis:names:tc:ebxml-regrep:xsd:rs:4.0" xmlns:ns7="urn:oasis:names:tc:ebxml-regrep:xsd:query:4.0"
xmlns:ns8="urn:oasis:names:tc:ebxml-regrep:xsd:spi:4.0">
<provider serviceType="WFS" name="MADISOGC">
<connection>
<!-- for OGC it's your FQDN
!!!!!PLACE YOUR URL HERE!!!!!!-->
<url>http://your.url.here:8085</url>
</connection>
<providerType>Point</providerType>
<projection type="LatLon">
<name>MadisLatLon</name>
<description>MADIS Test LatLon Coverage</description>
</projection>
</provider>
<agent xsi:type="ogcAgent" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<dateFormat>HHddMMMyyyy</dateFormat>
<layer name="madis">
<!-- Geographic constraint of madis layer data -->
<minx>-120.0</minx>
<maxx>-70.0</maxx>
<miny>20.0</miny>
<maxy>50.0</maxy>
<crs>crs:84</crs>
<!-- Registry Parameters in madis layer -->
<parameter name="timeObs" providerName="OBSTIME"
definition="Time of observation" dataType="POINT" baseType="String">
<levelType levelType="SFC" />
</parameter>
<parameter name="provider" providerName="PVDR"
definition="Provider Network" dataType="POINT" baseType="String">
<levelType levelType="SFC" />
</parameter>
<parameter name="sub_provider" providerName="SUBPVDR"
definition="Sub Provider Network" dataType="POINT" baseType="String">
<levelType levelType="SFC" />
</parameter>
<parameter name="restriction" providerName="RES"
definition="Restriction Level" dataType="POINT" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="dataset" providerName="DS"
definition="Dataset" dataType="POINT" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="obsLocation" providerName="obsLocation"
definition="Location of Observation" dataType="POINT" units="Degrees"
missingValue="-9999" fillValue="-9999" baseType="ObsLocation">
<levelType levelType="SFC" />
</parameter>
<parameter name="dewpoint" providerName="TD"
definition="Dewpoint Temperature" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="K">
<levelType levelType="SFC" />
</parameter>
<parameter name="dewpoint_qcd" providerName="TD_QCD"
definition="Dewpoint QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="dewpoint_qca" providerName="TD_QCA"
definition="Dewpoint QCD" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="dewpoint_qcr" providerName="TD_QCR"
definition="Dewpoint QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="rh" providerName="RH"
definition="Relative Humidity" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Integer" units="%">
<levelType levelType="SFC" />
</parameter>
<parameter name="rh_qcd" providerName="RH_QCD"
definition="Relative Humidity QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="rh_qca" providerName="RH_QCA"
definition="Relative Humidity QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="rh_qcr" providerName="RH_QCR"
definition="Relative Humidity QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="altimeter" providerName="ALTSE"
definition="Altimeter Setting" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Integer" units="Pa">
<levelType levelType="SFC" />
</parameter>
<parameter name="altimeter_qcd" providerName="ALTSE_QCD"
definition="Altimeter Setting QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="altimeter_qca" providerName="ALTSE_QCA"
definition="Altimeter Setting QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="altimeter_qcr" providerName="ALTSE_QCR"
definition="Altimeter Setting QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="temperature" providerName="T"
definition="Temperature" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="K">
<levelType levelType="SFC" />
</parameter>
<parameter name="temperature_qcd" providerName="T_QCD"
definition="Temperature QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="temperature_qca" providerName="T_QCA"
definition="Temperature QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="temperature_qcr" providerName="T_QCR"
definition="Temperature QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windDirection" providerName="DD"
definition="Wind Direction" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Integer" units="degrees">
<levelType levelType="SFC" />
</parameter>
<parameter name="windDirection_qcd" providerName="DD_QCD"
definition="Wind Direction QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="windDirection_qca" providerName="DD_QCA"
definition="Wind Direction QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windDirection_qcr" providerName="DD_QCR"
definition="Wind Direction QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windSpeed" providerName="FF"
definition="Wind Speed" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="degrees">
<levelType levelType="SFC" />
</parameter>
<parameter name="windSpeed_qcd" providerName="FF_QCD"
definition="Wind Speed QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="windSpeed_qca" providerName="FF_QCA"
definition="Wind Speed QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windSpeed_qcr" providerName="FF_QCR"
definition="Wind Speed QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="elevation" providerName="ELEV"
definition="Elevation" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="degrees">
<levelType levelType="SFC" />
</parameter>
<parameter name="elevation_qcd" providerName="ELEV_QCD"
definition="Elevation QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="elevation_qca" providerName="ELEV_QCA"
definition="Elevation QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="elevation_qcr" providerName="ELEV_QCR"
definition="Elevation QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="lon_qcd" providerName="LON_QCD"
definition="longitude QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="lon_qca" providerName="LON_QCA"
definition="Longitude QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="lon_qcr" providerName="LON_QCR"
definition="Longitude QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="lat_qcd" providerName="LAT_QCD"
definition="Latitude QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="lat_qca" providerName="LAT_QCA"
definition="Latitude QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="lat_qcr" providerName="LAT_QCR"
definition="Latitude QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipRate" providerName="PCPRATE"
definition="Precipitation Rate" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="in/hr">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipRate_qcd" providerName="PCPRATE_QCD"
definition="Precipitation Rate QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipRate_qca" providerName="PCPRATE_QCA"
definition="Precipitation Rate QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipRate_qcr" providerName="PCPRATE_QCR"
definition="Precipitation Rate QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windGust" providerName="FFGUST"
definition="Wind Gust" dataType="POINT"
missingValue="-9999" fillValue="-9999" baseType="Float" units="kts">
<levelType levelType="SFC" />
</parameter>
<parameter name="windGust_qcd" providerName="FFGUST_QCD"
definition="Wind Gust QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="windGust_qca" providerName="FFGUST_QCA"
definition="Wind Gust QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="windGust_qcr" providerName="FFGUST_QCR"
definition="Wind Gust QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipitalWater" providerName="PWV"
definition="Precipital Water Vapor" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="in">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipitalWater_qcd" providerName="PWV_QCD"
definition="Precipital Water Vapor QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipitalWater_qca" providerName="PWV_QCA"
definition="Precipital Water Vapor QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="precipitalWater_qcr" providerName="PWV_QCR"
definition="Precipital Water Vapor QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="pressure" providerName="P"
definition="Pressure" dataType="POINT" missingValue="-9999"
fillValue="-9999" baseType="Float" units="Pa">
<levelType levelType="SFC" />
</parameter>
<parameter name="pressure_qcd" providerName="P_QCD"
definition="Pressure QCD" dataType="POINT" baseType="QCD">
<levelType levelType="SFC" />
</parameter>
<parameter name="pressure_qca" providerName="P_QCA"
definition="Pressure QCA" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
<parameter name="pressure_qcr" providerName="P_QCR"
definition="Pressure QCR" dataType="POINT" missingValue="0"
fillValue="0" baseType="Integer">
<levelType levelType="SFC" />
</parameter>
</layer>
</agent>
</harvester>

10
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/NOMADS-harvester.xml
View file

@ -29,14 +29,6 @@
</connection> </connection>
</provider> </provider>
<!-- Registry hosts for harvester(defaults to localhost) -->
<!-- primary -->
<primaryRegistryHost>127.0.0.1</primaryRegistryHost>
<!-- secondary backup -->
<secondaryRegistryHost>127.0.0.1</secondaryRegistryHost>
<!-- secondary backup -->
<tertiaryRegistryHost>127.0.0.1</tertiaryRegistryHost>
<agent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="crawlAgent"> <agent xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="crawlAgent">
<crawlDir>/awips2/crawl</crawlDir> <crawlDir>/awips2/crawl</crawlDir>
<dateFormat>HHddMMMyyyy</dateFormat> <dateFormat>HHddMMMyyyy</dateFormat>
@ -56,7 +48,7 @@
<!-- Ingest new collections found by Seed Scans? --> <!-- Ingest new collections found by Seed Scans? -->
<ingestNew>true</ingestNew> <ingestNew>true</ingestNew>
<!-- listen to robots.txt denied directory lists? --> <!-- listen to robots.txt denied directory lists? -->
<useRobots>true</useRobots> <useRobots>false</useRobots>
<!-- (-1) is Unlimiited pages visited--> <!-- (-1) is Unlimiited pages visited-->
<maxSeedPages>-1</maxSeedPages> <maxSeedPages>-1</maxSeedPages>
<maxMainPages>1000</maxMainPages> <maxMainPages>1000</maxMainPages>

0
edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml → edexOsgi/com.raytheon.uf.edex.datadelivery.harvester/utility/common_static/base/datadelivery/harvester/OGC-harvester.xml.sample
View file

1
edexOsgi/com.raytheon.uf.edex.purgesrv/META-INF/MANIFEST.MF
View file

@ -10,3 +10,4 @@ Require-Bundle: com.raytheon.edex.common;bundle-version="1.11.26",
org.apache.commons.lang;bundle-version="2.3.0", org.apache.commons.lang;bundle-version="2.3.0",
com.raytheon.uf.common.status;bundle-version="1.12.1174", com.raytheon.uf.common.status;bundle-version="1.12.1174",
javax.persistence;bundle-version="1.0.0" javax.persistence;bundle-version="1.0.0"
Export-Package: com.raytheon.uf.edex.purgesrv

14
edexOsgi/com.raytheon.uf.edex.purgesrv/res/spring/purge-spring-impl.xml Normal file
View file

@ -0,0 +1,14 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
<bean id="purgeManager" class="com.raytheon.uf.edex.purgesrv.PurgeManager">
<property name="clusterLimit" value="${purge.clusterlimit}"/>
<property name="serverLimit" value="${purge.serverlimit}"/>
<property name="deadPurgeJobAge" value="${purge.deadjobage}"/>
<property name="purgeFrequency" value="${purge.frequency}"/>
<property name="fatalFailureCount" value="${purge.fatalfailurecount}"/>
<property name="purgeEnabled" value="${purge.enabled}"/>
</bean>
</beans>

12
edexOsgi/com.raytheon.uf.edex.purgesrv/res/spring/purge-spring.xml
View file

@ -5,16 +5,10 @@
http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd"> http://camel.apache.org/schema/spring http://camel.apache.org/schema/spring/camel-spring.xsd">
<bean id="purge" class="com.raytheon.uf.edex.purgesrv.PurgeSrv" /> <bean id="purge" class="com.raytheon.uf.edex.purgesrv.PurgeSrv">
<bean id="purgeManager" class="com.raytheon.uf.edex.purgesrv.PurgeManager" factory-method="getInstance"> <constructor-arg ref = "purgeManager"/>
<property name="clusterLimit" value="${purge.clusterlimit}"/>
<property name="serverLimit" value="${purge.serverlimit}"/>
<property name="deadPurgeJobAge" value="${purge.deadjobage}"/>
<property name="purgeFrequency" value="${purge.frequency}"/>
<property name="fatalFailureCount" value="${purge.fatalfailurecount}"/>
<property name="purgeEnabled" value="${purge.enabled}"/>
</bean> </bean>
<!-- Register database plugin --> <!-- Register database plugin -->
<bean id="purgeDatabaseProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties"> <bean id="purgeDatabaseProperties" class="com.raytheon.uf.edex.database.DatabasePluginProperties">
<property name="pluginFQN" value="com.raytheon.uf.edex.purgesrv" /> <property name="pluginFQN" value="com.raytheon.uf.edex.purgesrv" />

15
edexOsgi/com.raytheon.uf.edex.purgesrv/src/com/raytheon/uf/edex/purgesrv/PurgeJob.java
View file

@ -73,6 +73,8 @@ public class PurgeJob extends Thread {
/** Last time job has printed a timed out message */ /** Last time job has printed a timed out message */
private long lastTimeOutMessage = 0; private long lastTimeOutMessage = 0;
private PurgeManager purgeManager;
/** /**
* Creates a new Purge job for the specified plugin. * Creates a new Purge job for the specified plugin.
@ -82,11 +84,12 @@ public class PurgeJob extends Thread {
* @param purgeType * @param purgeType
* The type of purge to be executed * The type of purge to be executed
*/ */
public PurgeJob(String pluginName, PURGE_JOB_TYPE purgeType) { public PurgeJob(String pluginName, PURGE_JOB_TYPE purgeType, PurgeManager purgeManager) {
// Give the thread a name // Give the thread a name
this.setName("Purge-" + pluginName.toUpperCase() + "-Thread"); this.setName("Purge-" + pluginName.toUpperCase() + "-Thread");
this.pluginName = pluginName; this.pluginName = pluginName;
this.purgeType = purgeType; this.purgeType = purgeType;
this.purgeManager = purgeManager;
} }
public void run() { public void run() {
@ -143,7 +146,7 @@ public class PurgeJob extends Thread {
t = t.getCause(); t = t.getCause();
} }
} finally { } finally {
ClusterTask purgeLock = PurgeManager.getInstance().getPurgeLock(); ClusterTask purgeLock = purgeManager.getPurgeLock();
try { try {
/* /*
* Update the status accordingly if the purge failed or * Update the status accordingly if the purge failed or
@ -159,13 +162,11 @@ public class PurgeJob extends Thread {
} else { } else {
if (failed) { if (failed) {
status.incrementFailedCount(); status.incrementFailedCount();
if (status.getFailedCount() >= PurgeManager if (status.getFailedCount() >= purgeManager.getFatalFailureCount()) {
.getInstance().getFatalFailureCount()) {
PurgeLogger PurgeLogger
.logFatal( .logFatal(
"Purger for this plugin has reached or exceeded consecutive failure limit of " "Purger for this plugin has reached or exceeded consecutive failure limit of "
+ PurgeManager + purgeManager
.getInstance()
.getFatalFailureCount() .getFatalFailureCount()
+ ". Data will no longer being purged for this plugin.", + ". Data will no longer being purged for this plugin.",
pluginName); pluginName);
@ -188,7 +189,7 @@ public class PurgeJob extends Thread {
* This purger thread has exceeded the time out duration but * This purger thread has exceeded the time out duration but
* finally finished. Output a message and update the status * finally finished. Output a message and update the status
*/ */
int deadPurgeJobAge = PurgeManager.getInstance() int deadPurgeJobAge = purgeManager
.getDeadPurgeJobAge(); .getDeadPurgeJobAge();
Calendar purgeTimeOutLimit = Calendar.getInstance(); Calendar purgeTimeOutLimit = Calendar.getInstance();
purgeTimeOutLimit.setTimeZone(TimeZone.getTimeZone("GMT")); purgeTimeOutLimit.setTimeZone(TimeZone.getTimeZone("GMT"));

292
edexOsgi/com.raytheon.uf.edex.purgesrv/src/com/raytheon/uf/edex/purgesrv/PurgeManager.java
View file

@ -83,6 +83,7 @@ import com.raytheon.uf.edex.purgesrv.PurgeJob.PURGE_JOB_TYPE;
* Date Ticket# Engineer Description * Date Ticket# Engineer Description
* ------------ ---------- ----------- -------------------------- * ------------ ---------- ----------- --------------------------
* Apr 18, 2012 #470 bphillip Initial creation * Apr 18, 2012 #470 bphillip Initial creation
* Apr 11, 2013 #1959 dhladky Added method that only processes running plugins
* *
* </pre> * </pre>
* *
@ -141,163 +142,176 @@ public class PurgeManager {
private PurgeDao dao = new PurgeDao(); private PurgeDao dao = new PurgeDao();
private static PurgeManager instance = new PurgeManager();
public static PurgeManager getInstance() {
return instance;
}
/** /**
* Creates a new PurgeManager * Creates a new PurgeManager
*/ */
private PurgeManager() { protected PurgeManager() {
} }
/** /**
* Executes the purge routine * Executes the purge routine
*/ */
public void executePurge() { public void executePurge() {
if (!purgeEnabled) {
PurgeLogger.logWarn(
"Data purging has been disabled. No data will be purged.",
null);
return;
}
ClusterTask purgeMgrTask = getPurgeLock(); // Gets the list of plugins in ascending order by the last time they
// were purged
List<String> pluginList = dao.getPluginsByPurgeTime();
try { // check for any new plugins or database being purged and needing
// Prune the job map // entries recreated
Iterator<PurgeJob> iter = purgeJobs.values().iterator(); Set<String> availablePlugins = new HashSet<String>(PluginRegistry
while (iter.hasNext()) { .getInstance().getRegisteredObjects());
if (!iter.next().isAlive()) {
iter.remove();
}
}
Calendar purgeTimeOutLimit = Calendar.getInstance(); // Merge the lists
purgeTimeOutLimit.setTimeZone(TimeZone.getTimeZone("GMT")); availablePlugins.removeAll(pluginList);
purgeTimeOutLimit.add(Calendar.MINUTE, -deadPurgeJobAge);
Calendar purgeFrequencyLimit = Calendar.getInstance();
purgeFrequencyLimit.setTimeZone(TimeZone.getTimeZone("GMT"));
purgeFrequencyLimit.add(Calendar.MINUTE, -purgeFrequency);
// Gets the list of plugins in ascending order by the last time they if (availablePlugins.size() > 0) {
// were purged // generate new list with them at the beginning
List<String> pluginList = dao.getPluginsByPurgeTime(); List<String> newSortedPlugins = new ArrayList<String>(
availablePlugins);
Collections.sort(newSortedPlugins);
newSortedPlugins.addAll(pluginList);
pluginList = newSortedPlugins;
}
// check for any new plugins or database being purged and needing purgeRunner(pluginList);
// entries recreated }
Set<String> availablePlugins = new HashSet<String>(PluginRegistry
.getInstance().getRegisteredObjects());
// Merge the lists /**
availablePlugins.removeAll(pluginList); * The guts of the actual purge process
* @param availablePlugins
*/
protected void purgeRunner(List<String> pluginList) {
if (availablePlugins.size() > 0) { if (!purgeEnabled) {
// generate new list with them at the beginning PurgeLogger.logWarn(
List<String> newSortedPlugins = new ArrayList<String>( "Data purging has been disabled. No data will be purged.",
availablePlugins); null);
Collections.sort(newSortedPlugins); return;
newSortedPlugins.addAll(pluginList); }
pluginList = newSortedPlugins;
} ClusterTask purgeMgrTask = getPurgeLock();
try {
// Prune the job map
Iterator<PurgeJob> iter = purgeJobs.values().iterator();
while (iter.hasNext()) {
if (!iter.next().isAlive()) {
iter.remove();
}
}
Calendar purgeTimeOutLimit = Calendar.getInstance();
purgeTimeOutLimit.setTimeZone(TimeZone.getTimeZone("GMT"));
purgeTimeOutLimit.add(Calendar.MINUTE, -deadPurgeJobAge);
Calendar purgeFrequencyLimit = Calendar.getInstance();
purgeFrequencyLimit.setTimeZone(TimeZone.getTimeZone("GMT"));
purgeFrequencyLimit.add(Calendar.MINUTE, -purgeFrequency);
boolean canPurge = true; boolean canPurge = true;
int jobsStarted = 0; int jobsStarted = 0;
int maxNumberOfJobsToStart = Math.min( int maxNumberOfJobsToStart = Math.min(
clusterLimit clusterLimit
- dao.getRunningClusterJobs( - dao.getRunningClusterJobs(
purgeTimeOutLimit.getTime(), purgeTimeOutLimit.getTime(),
fatalFailureCount), serverLimit fatalFailureCount), serverLimit
- getNumberRunningJobsOnServer(purgeTimeOutLimit)); - getNumberRunningJobsOnServer(purgeTimeOutLimit));
for (String plugin : pluginList) {
try { if (!pluginList.isEmpty()) {
// initialize canPurge based on number of jobs started for (String plugin : pluginList) {
canPurge = jobsStarted < maxNumberOfJobsToStart; try {
PurgeJob jobThread = purgeJobs.get(plugin); // initialize canPurge based on number of jobs started
PurgeJobStatus job = dao.getJobForPlugin(plugin); canPurge = jobsStarted < maxNumberOfJobsToStart;
PurgeJob jobThread = purgeJobs.get(plugin);
PurgeJobStatus job = dao.getJobForPlugin(plugin);
if (job == null) { if (job == null) {
// no job in database, generate empty job // no job in database, generate empty job
try { try {
job = new PurgeJobStatus(); job = new PurgeJobStatus();
job.setPlugin(plugin); job.setPlugin(plugin);
job.setFailedCount(0); job.setFailedCount(0);
job.setRunning(false); job.setRunning(false);
job.setStartTime(new Date(0)); job.setStartTime(new Date(0));
dao.create(job); dao.create(job);
} catch (Throwable e) { } catch (Throwable e) {
PurgeLogger.logError( PurgeLogger.logError(
"Failed to create new purge job entry", "Failed to create new purge job entry",
plugin, e); plugin, e);
} }
} }
// Check to see if this job has met the fatal failure count // Check to see if this job has met the fatal failure
if (job.getFailedCount() >= fatalFailureCount) { // count
canPurge = false; if (job.getFailedCount() >= fatalFailureCount) {
PurgeLogger canPurge = false;
.logFatal( PurgeLogger
"Purger for this plugin has reached or exceeded consecutive failure limit of " .logFatal(
+ fatalFailureCount "Purger for this plugin has reached or exceeded consecutive failure limit of "
+ ". Data will no longer being purged for this plugin.", + fatalFailureCount
plugin); + ". Data will no longer being purged for this plugin.",
} plugin);
}
// is purge job currently running on this server // is purge job currently running on this server
if (jobThread != null) { if (jobThread != null) {
// job currently running on our server, don't start // job currently running on our server, don't start
// another // another
canPurge = false; canPurge = false;
if (purgeTimeOutLimit.getTimeInMillis() > jobThread if (purgeTimeOutLimit.getTimeInMillis() > jobThread
.getStartTime()) { .getStartTime()) {
jobThread.printTimedOutMessage(deadPurgeJobAge); jobThread.printTimedOutMessage(deadPurgeJobAge);
} }
} else { } else {
if (job.isRunning()) { if (job.isRunning()) {
// check if job has timed out // check if job has timed out
if (purgeTimeOutLimit.getTime().before( if (purgeTimeOutLimit.getTime().before(
job.getStartTime())) { job.getStartTime())) {
canPurge = false; canPurge = false;
} }
// else if no one else sets canPurge = false will // else if no one else sets canPurge = false
// start purging on this server // will
} else { // start purging on this server
// not currently running, check if need to be purged } else {
Date startTime = job.getStartTime(); // not currently running, check if need to be
if (startTime != null // purged
&& startTime.after(purgeFrequencyLimit Date startTime = job.getStartTime();
.getTime())) { if (startTime != null
canPurge = false; && startTime.after(purgeFrequencyLimit
} .getTime())) {
} canPurge = false;
} }
}
}
if (canPurge) { if (canPurge) {
purgeJobs.put(plugin, purgeExpiredData(plugin)); purgeJobs.put(plugin, purgeExpiredData(plugin));
jobsStarted++; jobsStarted++;
} }
} catch (Throwable e) { } catch (Throwable e) {
PurgeLogger PurgeLogger
.logError( .logError(
"An unexpected error occured during the purge job check for plugin", "An unexpected error occured during the purge job check for plugin",
plugin, e); plugin, e);
} }
} }
} catch (Throwable e) { }
PurgeLogger } catch (Throwable e) {
.logError( PurgeLogger
"An unexpected error occured during the data purge process", .logError(
StatusConstants.CATEGORY_PURGE, e); "An unexpected error occured during the data purge process",
} finally { StatusConstants.CATEGORY_PURGE, e);
// Unlock the purge task to allow other servers to run. } finally {
ClusterLockUtils.unlock(purgeMgrTask, false); // Unlock the purge task to allow other servers to run.
// PurgeLogger.logInfo(getPurgeStatus(true), null); ClusterLockUtils.unlock(purgeMgrTask, false);
} // PurgeLogger.logInfo(getPurgeStatus(true), null);
} }
}
@SuppressWarnings("unused") @SuppressWarnings("unused")
private String getPurgeStatus(boolean verbose) { private String getPurgeStatus(boolean verbose) {
@ -417,7 +431,7 @@ public class PurgeManager {
*/ */
public PurgeJob purgeExpiredData(String plugin) { public PurgeJob purgeExpiredData(String plugin) {
dao.startJob(plugin); dao.startJob(plugin);
PurgeJob job = new PurgeJob(plugin, PURGE_JOB_TYPE.PURGE_EXPIRED); PurgeJob job = new PurgeJob(plugin, PURGE_JOB_TYPE.PURGE_EXPIRED, this);
job.start(); job.start();
return job; return job;
} }
@ -433,7 +447,7 @@ public class PurgeManager {
*/ */
public PurgeJob purgeAllData(String plugin) { public PurgeJob purgeAllData(String plugin) {
dao.startJob(plugin); dao.startJob(plugin);
PurgeJob job = new PurgeJob(plugin, PURGE_JOB_TYPE.PURGE_ALL); PurgeJob job = new PurgeJob(plugin, PURGE_JOB_TYPE.PURGE_ALL, this);
job.start(); job.start();
return job; return job;
} }

13
edexOsgi/com.raytheon.uf.edex.purgesrv/src/com/raytheon/uf/edex/purgesrv/PurgeSrv.java
View file

@ -73,12 +73,15 @@ public class PurgeSrv {
/** The purge cron message */ /** The purge cron message */
public static final String PURGE_CRON = "PURGE_CRON"; public static final String PURGE_CRON = "PURGE_CRON";
private PurgeManager purgeManager;
/** /**
* Constructs a new PurgeSrv. This method verifies the metadata database has * Constructs a new PurgeSrv. This method verifies the metadata database has
* been constructed and exports the schema if necessary * been constructed and exports the schema if necessary
*/ */
public PurgeSrv() { public PurgeSrv(PurgeManager purgeManager) {
this.purgeManager = purgeManager;
} }
public void purgeCron() throws Exception { public void purgeCron() throws Exception {
@ -126,10 +129,10 @@ public class PurgeSrv {
purgeAllData(); purgeAllData();
} else if (message.startsWith(DELETE_PLUGIN_DATA)) { } else if (message.startsWith(DELETE_PLUGIN_DATA)) {
String pluginToPurge = message.replace(DELETE_PLUGIN_DATA, ""); String pluginToPurge = message.replace(DELETE_PLUGIN_DATA, "");
PurgeManager.getInstance().purgeExpiredData(pluginToPurge); purgeManager.purgeExpiredData(pluginToPurge);
} else if (message.startsWith(DELETE_ALL_PLUGIN_DATA)) { } else if (message.startsWith(DELETE_ALL_PLUGIN_DATA)) {
String pluginToPurge = message.replace(DELETE_ALL_PLUGIN_DATA, ""); String pluginToPurge = message.replace(DELETE_ALL_PLUGIN_DATA, "");
PurgeManager.getInstance().purgeAllData(pluginToPurge); purgeManager.purgeAllData(pluginToPurge);
} else if (message.equals(PURGE_CRON) } else if (message.equals(PURGE_CRON)
|| message.equals(DELETE_EXPIRED_DATA)) { || message.equals(DELETE_EXPIRED_DATA)) {
purgeExpiredData(); purgeExpiredData();
@ -160,7 +163,7 @@ public class PurgeSrv {
.getInstance().getRegisteredObjects()); .getInstance().getRegisteredObjects());
for (String pluginName : availablePlugins) { for (String pluginName : availablePlugins) {
if (PluginRegistry.getInstance().getRegisteredObject(pluginName) != null) { if (PluginRegistry.getInstance().getRegisteredObject(pluginName) != null) {
PurgeManager.getInstance().purgeAllData(pluginName); purgeManager.purgeAllData(pluginName);
} }
} }
PurgeLogger.logInfo("Purge All Data Completed at: " + new Date(), PurgeLogger.logInfo("Purge All Data Completed at: " + new Date(),
@ -183,7 +186,7 @@ public class PurgeSrv {
for (String pluginName : availablePlugins) { for (String pluginName : availablePlugins) {
if (PluginRegistry.getInstance().getRegisteredObject(pluginName) != null) { if (PluginRegistry.getInstance().getRegisteredObject(pluginName) != null) {
PurgeManager.getInstance().purgeExpiredData(pluginName); purgeManager.purgeExpiredData(pluginName);
} }
} }
} }