xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Omaha #4783 - implemented synopticToShefFilter and synopticToShefRun.

Browse source 
amend: ensure that defaults will be used in synoptic to shef when an alternative is not specified.
amend: additional code cleanup

Change-Id: Ib247052a5478c04d598fe4a1770194f6677b671b

Former-commit-id: 193624b14008dab771d4b72f277c4de4d6cd2534
This commit is contained in:
Bryan Kowal 2015-10-28 19:35:43 -05:00
parent 49565633cd
commit af8a0814fb
9 changed files with 723 additions and 299 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
edexOsgi/com.raytheon.edex.plugin.shef/res/spring/shef-ingest.xml
View file

@ -30,7 +30,7 @@
</bean>
<bean id="synopticToShefFilter"
class="com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter">
class="com.raytheon.edex.transform.shef.SynopticToShefFilter">
<constructor-arg type="java.lang.String" value="synopticToShefFilter.xml" />
<constructor-arg type="java.lang.String" value="SITE" />
</bean>

183
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefFilter.java Normal file
View file

@ -0,0 +1,183 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.transform.shef;
import static com.raytheon.uf.common.localization.LocalizationContext.LocalizationType.EDEX_STATIC;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractObsFilter;
import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
import com.raytheon.uf.edex.decodertools.core.filterimpl.RadiusFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.RectFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.StationIdFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement;
/**
* Abstraction of a Shef Filter.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 28, 2015 4783 bkowal Initial creation
*
* </pre>
*
* @author bkowal
* @version 1.0
*/
public abstract class AbstractShefFilter {
private static final String ERROR_1_FMT = "Could not create {%s} context for file \"%s\"";
private static final String ERROR_2_FMT = "File %s does not exist";
protected static final String METAR_CFG = "metar.cfg";
protected static final String DUMMY_FILTER_NAME = "Created Pass-All filter";
public static final String FILTERS_DIR = "plugin-filters";
protected final Log logger = LogFactory.getLog(getClass());
private String filterConfigFile = null;
protected AbstractShefFilter() {
this.init();
}
protected AbstractShefFilter(String configFile, String localContext,
final Class<? extends AbstractShefFilter> shefFilterClass) {
this.init();
this.filterConfigFile = configFile;
this.readConfig(localContext, shefFilterClass);
}
private void readConfig(String localContext,
final Class<? extends AbstractShefFilter> shefFilterClass) {
try {
/*
* Retrieve and verify the existence of the configuration file.
*/
IPathManager manager = PathManagerFactory.getPathManager();
if (manager == null) {
throw new Exception("Failed to construct a Path Manager.");
}
LocalizationContext context = manager.getContext(EDEX_STATIC,
LocalizationLevel.valueOf(localContext));
if (context == null) {
throw new Exception(String.format(ERROR_1_FMT, localContext,
this.filterConfigFile));
}
Path configPath = Paths.get(
manager.getFile(context, FILTERS_DIR).getAbsolutePath())
.resolve(filterConfigFile);
if (Files.exists(configPath) == false) {
throw new Exception(String.format(ERROR_2_FMT,
configPath.toString()));
}
/*
* Read the configuration file.
*/
JAXBManager jaxb = new JAXBManager(PluginDataObjectFilter.class,
shefFilterClass, RadiusFilterElement.class,
RectFilterElement.class, StationIdFilterElement.class,
WMOHeaderFilterElement.class);
byte[] data = Files.readAllBytes(configPath);
Object obj = jaxb.unmarshalFromXml(new String(data));
if (shefFilterClass.isInstance(obj)
|| obj instanceof PluginDataObjectFilter) {
this.buildRun(obj, this.filterConfigFile);
} else {
throw new Exception("Read unexpected config data type: "
+ obj.getClass().getName());
}
} catch (Exception e) {
logger.error("Failed to process filter configuration.", e);
this.createDummyFilter();
}
}
protected PluginDataObject filterARun(PluginDataObject report,
List<AbstractFilterElement> filterElements) {
if (report != null) {
PluginDataObject r = null;
boolean keep = true;
for (AbstractFilterElement element : filterElements) {
r = element.filter(report);
/*
* Only allow keep to be set to false. Once false it stays that
* way.
*/
if (AbstractObsFilter.INCLUDE_TYPE.equals(element
.getFilterType())) {
// Did the filter pass?
if (r == null) {
// If we fail an element, exit now.
keep = false;
break;
}
} else if (AbstractObsFilter.EXCLUDE_TYPE.equals(element
.getFilterType())) {
if (r != null) {
// There was a match, so we want to remove this
// item.
keep = false;
// And there's no reason for further checks.
break;
}
}
}
if (keep) {
report = r;
} else {
report = null;
}
}
return report;
}
protected abstract void init();
protected abstract void buildRun(final Object obj, final String configFile);
protected abstract void createDummyFilter();
public abstract PluginDataObject[] filter(PluginDataObject[] reports);
}

3
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/AbstractShefTransformer.java
View file

@ -54,6 +54,7 @@ import com.raytheon.uf.common.wmo.WMOTimeParser;
* 20120918 1185 jkorman Added save to archive capability.
* May 14, 2014 2536 bclement moved WMO Header to common, removed TimeTools usage
* Jul 01, 2015 16903 lbousaidi added routine for synoptic data
* Oct 29, 2015 4783 bkowal Made {@link #metar2ShefOptions} protected.
* </pre>
*
* @author jkorman
@ -123,7 +124,7 @@ public abstract class AbstractShefTransformer<T extends PluginDataObject>
private String commandLineOptions = null;
private String metar2ShefOptions = null;
protected String metar2ShefOptions = null;
private boolean archiveEnabled = false;

199
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefFilter.java
View file

@ -19,40 +19,23 @@
**/
package com.raytheon.edex.transform.shef;
import static com.raytheon.uf.common.localization.LocalizationContext.LocalizationType.EDEX_STATIC;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.localization.IPathManager;
import com.raytheon.uf.common.localization.LocalizationContext;
import com.raytheon.uf.common.localization.LocalizationContext.LocalizationLevel;
import com.raytheon.uf.common.localization.PathManagerFactory;
import com.raytheon.uf.common.ohd.AppsDefaults;
import com.raytheon.uf.common.serialization.JAXBManager;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractObsFilter;
import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
import com.raytheon.uf.edex.decodertools.core.filterimpl.RadiusFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.RectFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.StationIdFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement;
/**
* Use information in metarToShefFilter.xml, MetarToShefFilter filters out the
@ -68,6 +51,7 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement;
* 2/13/2013 1584 mpduff Fix creation of "dummy" config.
* 08/08/2013 16408 wkwock Use different metar.cfg file and options
* 09/09/2014 3548 mapeters Replaced SerializationUtil usage with JAXBManager.
* 10/28/2015 4783 bkowal Refactored and abstracted into {@link AbstractShefFilter}.
*
* </pre>
*
@ -77,92 +61,22 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.WMOHeaderFilterElement;
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class MetarToShefFilter {
public class MetarToShefFilter extends AbstractShefFilter {
@XmlElement
@DynamicSerializeElement
protected List<MetarToShefRun> metarToShefRun = new ArrayList<MetarToShefRun>();
private static final String ERROR_1_FMT = "Could not create {%s} context for file \"%s\"";
private static final String ERROR_2_FMT = "File %s does not exist";
private static final String METAR_CFG = "metar.cfg";
public static final String FILTERS_DIR = "plugin-filters";
protected List<MetarToShefRun> metarToShefRun;
private final String metarToShefOptions = AppsDefaults.getInstance()
.getToken("metar2shef_options");
private final Log logger = LogFactory.getLog(getClass());
private String filterConfigFile = null;
.getToken(AbstractShefTransformer.METAR_2_SHEF_OPT);
/**
* Empty constructor for {@link DynamicSerialize}.
*/
public MetarToShefFilter() {
}
public MetarToShefFilter(String configFile, String localContext) {
filterConfigFile = configFile;
try {
File filterDir = null;
IPathManager manager = PathManagerFactory.getPathManager();
if (manager != null) {
LocalizationContext context = manager.getContext(EDEX_STATIC,
LocalizationLevel.valueOf(localContext));
if (context != null) {
filterDir = manager.getFile(context, FILTERS_DIR);
if (filterDir.exists()) {
File srcFile = new File(filterDir, filterConfigFile);
byte[] data = new byte[(int) srcFile.length()];
InputStream stream = getInputStream(srcFile);
stream.read(data);
stream.close();
JAXBManager jaxb = new JAXBManager(
PluginDataObjectFilter.class,
MetarToShefFilter.class,
RadiusFilterElement.class,
RectFilterElement.class,
StationIdFilterElement.class,
WMOHeaderFilterElement.class);
Object obj = jaxb.unmarshalFromXml(new String(data));
if (obj instanceof PluginDataObjectFilter) {
logger.info("Found " + filterConfigFile
+ " is PluginDataObjectFilter");
PluginDataObjectFilter pdof = (PluginDataObjectFilter) obj;
MetarToShefRun mtsr = new MetarToShefRun();
mtsr.setConfigFileName(METAR_CFG);
mtsr.setMetarToShefOptions(metarToShefOptions);
mtsr.setFilterElements(pdof.getFilterElements());
mtsr.setFilterName(pdof.getFilterName());
this.metarToShefRun.add(mtsr);
} else if (obj instanceof MetarToShefFilter) {
MetarToShefFilter filter = (MetarToShefFilter) obj;
this.metarToShefRun = filter.metarToShefRun;
logger.info("Found " + filterConfigFile
+ " is MetarToShefFilter");
} else {
logger.error("Found " + filterConfigFile + " is "
+ obj.getClass().getCanonicalName());
createDummyFilter();
}
} else {
logger.error(String.format(ERROR_2_FMT,
filterDir.getPath()));
createDummyFilter();
}
} else {
logger.error(String.format(ERROR_1_FMT, localContext,
configFile));
createDummyFilter();
}
} else {
// Could not create PathManager
}
} catch (Exception e) {
logger.error("Error creating filter from " + filterConfigFile, e);
createDummyFilter();
}
super(configFile, localContext, MetarToShefFilter.class);
for (MetarToShefRun mtsr : metarToShefRun) {
logger.info("Filter name = " + mtsr.getFilterName()
@ -170,67 +84,56 @@ public class MetarToShefFilter {
}
}
private PluginDataObject filterARun(PluginDataObject report,
List<AbstractFilterElement> filterElements) {
if (report != null) {
PluginDataObject r = null;
boolean keep = true;
for (AbstractFilterElement element : filterElements) {
r = element.filter(report);
@Override
public void init() {
this.metarToShefRun = new ArrayList<>();
}
// Only allow keep to be set to true. Once true it stays
// that way.
if (AbstractObsFilter.INCLUDE_TYPE.equals(element
.getFilterType())) {
// Did the filter pass?
if (r == null) {
// If we fail an element, exit now.
keep = false;
break;
}
} else if (AbstractObsFilter.EXCLUDE_TYPE.equals(element
.getFilterType())) {
if (r != null) {
// There was a match, so we want to remove this
// item.
keep = false;
// And there's no reason for further checks.
break;
}
}
}
if (keep) {
report = r;
} else {
report = null;
}
@Override
protected void buildRun(final Object obj, final String configFile) {
if (obj instanceof PluginDataObjectFilter) {
logger.debug("Found " + configFile + " is PluginDataObjectFilter");
PluginDataObjectFilter pdof = (PluginDataObjectFilter) obj;
MetarToShefRun mtsr = new MetarToShefRun();
mtsr.setConfigFileName(METAR_CFG);
mtsr.setMetarToShefOptions(metarToShefOptions);
mtsr.setFilterElements(pdof.getFilterElements());
mtsr.setFilterName(pdof.getFilterName());
this.metarToShefRun.add(mtsr);
} else if (obj instanceof MetarToShefFilter) {
logger.debug("Found " + configFile + " is MetarToShefFilter");
MetarToShefFilter filter = (MetarToShefFilter) obj;
this.metarToShefRun = filter.metarToShefRun;
}
return report;
}
/**
* Apply the list of filters against given input data.
*
*/
// @Override
@Override
public PluginDataObject[] filter(PluginDataObject[] reports) {
HashMap<String,MetarToShefRun> matchList = new HashMap<String,MetarToShefRun>();
ArrayList<PluginDataObject> reportList = new ArrayList<PluginDataObject> ();
Map<Integer, MetarToShefRun> matchList = new HashMap<>();
List<PluginDataObject> reportList = new ArrayList<PluginDataObject>();
for (PluginDataObject report : reports) {
for (MetarToShefRun mtsr : metarToShefRun) {
PluginDataObject resultRpt = filterARun(report, mtsr.getFilterElements());
if (resultRpt != null ) {
reportList.add(resultRpt);
matchList.put(resultRpt.getDataURI(), mtsr);
MetarToShefTransformer.setMatchList (matchList);
PluginDataObject resultRpt = filterARun(report,
mtsr.getFilterElements());
if (resultRpt != null) {
reportList.add(resultRpt);
matchList.put(resultRpt.getId(), mtsr);
break;
}
}
}
return (PluginDataObject[])reportList.toArray(new PluginDataObject[reportList.size()]);
if (matchList.isEmpty() == false) {
MetarToShefTransformer.setMatchList(matchList);
}
return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]);
}
private void createDummyFilter() {
@Override
protected void createDummyFilter() {
MetarToShefRun mtsr = new MetarToShefRun();
mtsr.setConfigFileName(METAR_CFG);
mtsr.setMetarToShefOptions(metarToShefOptions);
@ -244,26 +147,10 @@ public class MetarToShefFilter {
};
dummy.setFilterType(AbstractObsFilter.INCLUDE_TYPE);
mtsr.getFilterElements().add(dummy);
mtsr.setFilterName("Created Pass-All filter");
mtsr.setFilterName(DUMMY_FILTER_NAME);
this.metarToShefRun.add(mtsr);
}
/**
*
* @param file
* @return
*/
private static FileInputStream getInputStream(File file) {
FileInputStream fis = null;
try {
fis = new FileInputStream(file);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
return fis;
}
public void addMetarToShefRun(MetarToShefRun element) {
metarToShefRun.add(element);
}

38
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefRun.java
View file

@ -1,4 +1,5 @@
package com.raytheon.edex.transform.shef;
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
@ -27,7 +28,6 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import com.raytheon.uf.common.serialization.ISerializableObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
@ -36,28 +36,30 @@ import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
* MetarToShefRun is for the metarToShefRun tag in Metar2ShefFilter tag
*
* <pre>
*
*
* SOFTWARE HISTORY
*
*
* Date Ticket# Engineer Description
* ---------- ------- -------- --------------------------
* 1/10/2013 15497 wkwock Initial creation
*
* 10/28/2015 4783 bkowal Removed ISerializableObject.
*
* </pre>
*
*
* @author wkwock
* @version 1.0
* @version 1.0
*/
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class MetarToShefRun implements ISerializableObject{
public class MetarToShefRun {
public static final String INCLUDE_TYPE = "INCLUDE";
public static final String EXCLUDE_TYPE = "EXCLUDE";
@XmlElement
@DynamicSerializeElement
private String configFileName;
@XmlElement
@DynamicSerializeElement
private String metarToShefOptions;
@ -69,14 +71,14 @@ public class MetarToShefRun implements ISerializableObject{
@XmlElement
@DynamicSerializeElement
private String filterName;
/**
*
* @param filterFile
*/
void createFilter(File filterFile) {
}
/**
*
* @return
@ -84,13 +86,13 @@ public class MetarToShefRun implements ISerializableObject{
public String getConfigFileName() {
return configFileName;
}
/**
*
* @param name
*/
public void setConfigFileName(String name) {
configFileName = name;
configFileName = name;
}
/**
@ -100,13 +102,13 @@ public class MetarToShefRun implements ISerializableObject{
public String getMetarToShefOptions() {
return metarToShefOptions;
}
/**
*
* @param name
*/
public void setMetarToShefOptions(String name) {
metarToShefOptions = name;
metarToShefOptions = name;
}
/**
@ -123,7 +125,7 @@ public class MetarToShefRun implements ISerializableObject{
public List<AbstractFilterElement> getFilterElements() {
return filterElements;
}
/**
*
* @param elements
@ -131,7 +133,7 @@ public class MetarToShefRun implements ISerializableObject{
public void setFilterElements(List<AbstractFilterElement> elements) {
filterElements = elements;
}
/**
*
* @return
@ -139,7 +141,7 @@ public class MetarToShefRun implements ISerializableObject{
public String getFilterName() {
return filterName;
}
/**
*
* @param name

189
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/MetarToShefTransformer.java
View file

@ -23,6 +23,7 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -76,36 +77,41 @@ public class MetarToShefTransformer extends
private static final String SENS_TYPE_AUTO = " AUTO";
private static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
private static final DateFormat dateFormat = new SimpleDateFormat(
"yyyy/MM/dd HH:mm:ss");
static {
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
}
private static final int P1_MIN = 50;
private static final int P2_MAX = 5;
private static final int P1_MIN = 50;
private static final int P2_MAX = 5;
/*
* Max list for optionsList and matchList to prevent memory leak.
* (500 or any big number should be fine)
* Max list for optionsList and matchList to prevent memory leak. (500 or
* any big number should be fine)
*/
private static final int MAX_LIST = 500;
/*
* MetarToShefRun list which has metar.cfg file and options to used by conversion
* MetarToShefRun list which has metar.cfg file and options to used by
* conversion
*/
private static HashMap<String,MetarToShefRun> mtsrList = new HashMap<String,MetarToShefRun>();
private static HashMap<Integer, MetarToShefRun> mtsrList = new HashMap<>();
/*
* List of metar.cfg files + options.
* List of metar.cfg files + options.
*/
private HashMap<String,ObsToSHEFOptions> optionsList = new HashMap<String,ObsToSHEFOptions>();
private ObsToSHEFOptions defaultOptions =null;
private HashMap<String, ObsToSHEFOptions> optionsList = new HashMap<String, ObsToSHEFOptions>();
private ObsToSHEFOptions defaultOptions = null;
/**
* Construct an instance of this transformer.
* @param cmdLine Command line options that may be used if these
* options are not present in the Apps_defaults.
*
* @param cmdLine
* Command line options that may be used if these options are not
* present in the Apps_defaults.
*/
public MetarToShefTransformer(String cmdLine) {
super(cmdLine, WMO_HEADER_FMT);
@ -113,11 +119,16 @@ public class MetarToShefTransformer extends
}
/**
* Attempt to transform a single metar observation into SHEF encoded observations.
* @param report A metar report to encode.
* @param The system headers associated with the original metar message.
* Attempt to transform a single metar observation into SHEF encoded
* observations.
*
* @param report
* A metar report to encode.
* @param The
* system headers associated with the original metar message.
* @return The encoded SHEF report as a byte array. May return an empty
* array if the report should not have been encoded or some error occured.
* array if the report should not have been encoded or some error
* occured.
* @throws TransformerException
*/
@Override
@ -155,7 +166,8 @@ public class MetarToShefTransformer extends
place = 2;
StringBuilder sb = makeWMOHeader(openWMOMessage(200), stnId,
headers, hdr);
String fileName = makeWMOHeader(new StringBuilder(20), stnId, headers, hdr).toString().trim().replace(' ', '_');
String fileName = makeWMOHeader(new StringBuilder(20), stnId,
headers, hdr).toString().trim().replace(' ', '_');
place = 3;
startMessageLine(sb);
@ -177,7 +189,7 @@ public class MetarToShefTransformer extends
sb.append(": Observation time = ");
sb.append(report.getDataTime());
sb.append(" System time= ");
synchronized(dateFormat) {
synchronized (dateFormat) {
sb.append(dateFormat.format(nowCalendar.getTime()));
}
}
@ -242,7 +254,7 @@ public class MetarToShefTransformer extends
place = 1;
if (options.isOptStripICAO()) {
lineHdr.append(stnId.substring(1));
} else { // Only for ALASKA region
} else { // Only for ALASKA region
lineHdr.append(stnId);
}
@ -263,10 +275,10 @@ public class MetarToShefTransformer extends
lineHdr.append(": ");
c = report.getTimeObs();
if(METAR.equals(report.getReportType())) {
if (METAR.equals(report.getReportType())) {
c = checkTimeRounding(c, options, report);
}
place = 2;
if (options.isOptCentury()) {
lineHdr.append(String.format(SHEF_OBS_DATEY2K_FMT, c));
@ -294,7 +306,7 @@ public class MetarToShefTransformer extends
}
}
buffer = writeObs(buffer, rptText.toString());
// Now grab the remarks. This is all we need in the formatter
// section.
place = 4;
@ -303,8 +315,8 @@ public class MetarToShefTransformer extends
String reportText = null;
if (m.find()) {
reportText = rptText.substring(m.start());
if(Utilities.isAutoASOS(reportText)) {
options.setGeneralProperty(Utilities.IS_ASOS,"T");
if (Utilities.isAutoASOS(reportText)) {
options.setGeneralProperty(Utilities.IS_ASOS, "T");
}
} else {
reportText = "";
@ -329,16 +341,17 @@ public class MetarToShefTransformer extends
return buffer;
}
private static Calendar checkTimeRounding(Calendar c, ObsToSHEFOptions options, MetarRecord wxReport) {
if(c != null) {
if(options != null) {
if(options.isOptRoundObsTime()) {
private static Calendar checkTimeRounding(Calendar c,
ObsToSHEFOptions options, MetarRecord wxReport) {
if (c != null) {
if (options != null) {
if (options.isOptRoundObsTime()) {
String id = wxReport.getStationId();
Integer durTime = null;
Integer pcReset = options.getPCReset(id);
// !null means that a reset is defined for this station.
if(pcReset == null) {
if (pcReset == null) {
if (options.isOptStripICAO()) {
// Try again with 3 character identifier.
pcReset = options.getPCReset(id.substring(1));
@ -349,31 +362,31 @@ public class MetarToShefTransformer extends
// Get the tolerance value
Integer tol = options.getOptPCT();
int obMinute = wxReport.getTimeObs().get(Calendar.MINUTE);
int obMinute = wxReport.getTimeObs().get(
Calendar.MINUTE);
durTime = (obMinute + 60 - pcReset) % 60;
if((durTime <= tol) || (Math.abs(durTime - 60) <= tol)) {
if ((durTime <= tol) || (Math.abs(durTime - 60) <= tol)) {
durTime = 60;
}
}
// if null then the station does not have pcreset applied.
if(durTime == null) {
if (durTime == null) {
int min = c.get(Calendar.MINUTE);
int deltaHour = 0;
if(min >= 30) {
if (min >= 30) {
deltaHour = 1;
}
min = 0;
c.set(Calendar.MINUTE,min);
c.add(Calendar.HOUR_OF_DAY,deltaHour);
c.set(Calendar.MINUTE, min);
c.add(Calendar.HOUR_OF_DAY, deltaHour);
}
}
}
}
return c;
}
/**
* Should this reports data be encoded?
*
@ -385,32 +398,35 @@ public class MetarToShefTransformer extends
@Override
protected boolean encodeThisStation(MetarRecord report) {
boolean encodeOk = true;
int mnTime = report.getTimeObs().get(Calendar.MINUTE);
if(options.isOptSpeci()) {
if((mnTime > P2_MAX) && (mnTime < P1_MIN)) {
if (options.isOptSpeci()) {
if ((mnTime > P2_MAX) && (mnTime < P1_MIN)) {
encodeOk = report.getPrecip1Hour() >= 0;
}
encodeOk &= "METAR".equals(report.getReportType());
}
String s = report.getStationId();
return (encodeOk & options.checkName(s));
}
/**
* Reformats observation text into a SHEF comment.
* @param sb Buffer to receive the formatted data.
* @param obs The observation to format.
*
* @param sb
* Buffer to receive the formatted data.
* @param obs
* The observation to format.
* @return The formatted data.
*/
public static StringBuilder writeObs(StringBuilder sb, String obs) {
boolean startLine = true;
int count = 0;
String indent = "";
for(int i = 0;i < obs.length();i++) {
if(startLine) {
for (int i = 0; i < obs.length(); i++) {
if (startLine) {
startMessageLine(sb);
sb.append(":");
sb.append(indent);
@ -418,14 +434,14 @@ public class MetarToShefTransformer extends
}
char c = obs.charAt(i);
count++;
if((c == '\r') || (c == '\n')) {
if ((c == '\r') || (c == '\n')) {
c = ' ';
}
if(c != ' ') {
if (c != ' ') {
sb.append(c);
} else {
// we have a space
if(count >= 65) {
if (count >= 65) {
startLine = true;
count = 4;
indent = " ";
@ -438,39 +454,46 @@ public class MetarToShefTransformer extends
}
public final byte[] transformMetar(MetarRecord report, Headers headers)
throws TransformerException {
ObsToSHEFOptions tmpOptions=null;
MetarToShefRun mtsr = mtsrList.get(report.getDataURI());
if (mtsr==null) {
tmpOptions = defaultOptions;
} else {
tmpOptions = optionsList.get(mtsr.getConfigFileName()+mtsr.getMetarToShefOptions());
if (tmpOptions==null) {
//just to prevent t memory leak
if (optionsList.size()>MAX_LIST) {
optionsList.clear();
}
tmpOptions = new ObsToSHEFOptions(mtsr.getConfigFileName(),mtsr.getMetarToShefOptions(),true);
optionsList.put(mtsr.getConfigFileName()+mtsr.getMetarToShefOptions(), tmpOptions);
}
mtsrList.remove(report.getDataURI());
}
options=tmpOptions;
logger.info("Metar to SHEF for "+report.getStationId()+" use config file: "+options.getCfgFileName()+" with options:"+mtsr.getMetarToShefOptions());
configureArchiveDir();
throws TransformerException {
ObsToSHEFOptions tmpOptions = null;
MetarToShefRun mtsr = mtsrList.get(report.getId());
if (mtsr == null) {
tmpOptions = defaultOptions;
} else {
tmpOptions = optionsList.get(mtsr.getConfigFileName()
+ mtsr.getMetarToShefOptions());
if (tmpOptions == null) {
// just to prevent t memory leak
if (optionsList.size() > MAX_LIST) {
optionsList.clear();
}
tmpOptions = new ObsToSHEFOptions(mtsr.getConfigFileName(),
mtsr.getMetarToShefOptions(), true);
optionsList
.put(mtsr.getConfigFileName()
+ mtsr.getMetarToShefOptions(), tmpOptions);
}
mtsrList.remove(report.getId());
}
options = tmpOptions;
logger.info("Metar to SHEF for " + report.getStationId()
+ " use config file: " + options.getCfgFileName()
+ " with options:" + mtsr.getMetarToShefOptions());
configureArchiveDir();
return transformReport(report, headers);
return transformReport(report, headers);
}
/*
* set matchList
*/
public static void setMatchList (HashMap<String,MetarToShefRun> matchLst) {
//should add to the list and remove after use. clear if reach certain big number
if (mtsrList.size()>MAX_LIST) {
mtsrList.clear();
}
mtsrList.putAll(matchLst);
public static void setMatchList(Map<Integer, MetarToShefRun> matchLst) {
// should add to the list and remove after use. clear if reach certain
// big number
if (mtsrList.size() > MAX_LIST) {
mtsrList.clear();
}
mtsrList.putAll(matchLst);
}
}

134
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SMToShefTransformer.java
View file

@ -22,15 +22,17 @@ package com.raytheon.edex.transform.shef;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;
import javax.xml.transform.TransformerException;
import com.raytheon.edex.esb.Headers;
import com.raytheon.edex.transform.shef.obs.ObsToSHEFOptions;
import com.raytheon.edex.transform.shef.obs.SHEF_Obs_Codes;
import com.raytheon.edex.transform.shef.obs.SHEF_SM_Codes;
import com.raytheon.uf.common.dataplugin.sfcobs.ObsCommon;
import com.raytheon.uf.common.wmo.WMOHeader;
import com.raytheon.uf.common.wmo.WMOTimeParser;
import com.raytheon.uf.edex.decodertools.core.DecoderTools;
import com.raytheon.uf.edex.decodertools.core.IDecoderConstants;
@ -51,6 +53,8 @@ import com.raytheon.uf.edex.decodertools.core.IDecoderConstants;
* May 14, 2014 2536 bclement moved WMO Header to common, removed TimeTools usage
* Jul 01, 2015 16903 lbousaidi fixed WMO header in ingest log and product_id inserted
* into ihfs database
* Oct 28, 2015 4783 bkowal Allow {@link SynopticToShefRun}s to override the
* default configuration.
* </pre>
*
* @author jkorman
@ -61,13 +65,22 @@ public class SMToShefTransformer extends AbstractShefTransformer<ObsCommon> {
private static final String WMO_HEADER_FMT = CRCRLF
+ "SRXX99 %4s %2$td%2$tH%2$tM";
private static final int DT_SIZE = 6;
private static final int CCC_SIZE = 6;
private static final int MAX_LIST = 500;
private static Map<Integer, SynopticToShefRun> matchMap = new HashMap<>();
private final ObsToSHEFOptions defaultOptions;
private HashMap<String, ObsToSHEFOptions> optionsCacheMap = new HashMap<>();
private static final DateFormat dateFormat = new SimpleDateFormat("ddHHmm");
static {
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
}
/**
@ -75,6 +88,7 @@ public class SMToShefTransformer extends AbstractShefTransformer<ObsCommon> {
*/
public SMToShefTransformer(String cmdLine) {
super(cmdLine, WMO_HEADER_FMT);
this.defaultOptions = this.options;
}
/**
@ -90,48 +104,81 @@ public class SMToShefTransformer extends AbstractShefTransformer<ObsCommon> {
public byte[] transformReport(ObsCommon report, Headers headers)
throws TransformerException {
ObsToSHEFOptions tmpOptions = null;
SynopticToShefRun run = matchMap.get(report.getId());
if (run == null) {
tmpOptions = this.defaultOptions;
} else {
final String optionsKey = run.getConfigFileName();
tmpOptions = this.optionsCacheMap.get(optionsKey);
if (tmpOptions == null) {
if (this.optionsCacheMap.size() > MAX_LIST) {
this.optionsCacheMap.clear();
}
/*
* {@link #metar2ShefOptions} contain the parameters that were
* used when constructing the default {@link ObsToSHEFOptions}
* options.
*/
if (run.getConfigFileName() == null) {
tmpOptions = new ObsToSHEFOptions(this.metar2ShefOptions,
true);
} else {
tmpOptions = new ObsToSHEFOptions(run.getConfigFileName(),
this.metar2ShefOptions, true);
}
this.optionsCacheMap.put(optionsKey, tmpOptions);
}
matchMap.remove(report.getId());
}
this.options = tmpOptions;
logger.info("Synoptic to SHEF for " + report.getStationId()
+ " use config file: " + options.getCfgFileName());
// Transformed Synoptic PluginDataObject to SHEF
byte[] result = null;
byte[] result = null;
try {
// Currently returns false, so nothing is encoded at this time.
if (encodeThisStation(report)) {
//get the id and use the alias to change digits to letters.
String stnId = report.getStationId();
if (options.isOptCheckAliasId()) {
stnId = options.checkAlias(stnId);
// get the id and use the alias to change digits to letters.
String stnId = report.getStationId();
if (options.isOptCheckAliasId()) {
stnId = options.checkAlias(stnId);
}
// make header for ingest log file printout
String YYGGgg = report.getWmoHeader().substring(12, 12 + DT_SIZE);
String ccc = report.getWmoHeader().substring(6, 6 + CCC_SIZE);
StringBuilder sb = makeSynHeader(openWMOMessage(200),stnId ,headers , YYGGgg);
String fileName = makeSynHeader(new StringBuilder(20),stnId,headers, YYGGgg)
.toString().trim().replace(' ', '_');
startMessageLine(sb);
if (ccc != null) {
if (ccc.length() > 3) {
ccc = ccc.substring(ccc.length() - 4).trim();
// make header for ingest log file printout
String YYGGgg = report.getWmoHeader().substring(12,
12 + DT_SIZE);
String ccc = report.getWmoHeader().substring(6, 6 + CCC_SIZE);
StringBuilder sb = makeSynHeader(openWMOMessage(200), stnId,
headers, YYGGgg);
String fileName = makeSynHeader(new StringBuilder(20), stnId,
headers, YYGGgg).toString().trim().replace(' ', '_');
startMessageLine(sb);
if (ccc != null) {
if (ccc.length() > 3) {
ccc = ccc.substring(ccc.length() - 4).trim();
}
}
sb.append(ccc);
sb.append(METAR_2_SHEF_NNN);
if (stnId.length() == 4) {
sb.append(stnId.substring(1));
} else if (stnId.length() == 3) {
sb.append(stnId);
}
startMessageLine(sb);
startMessageLine(sb).append(
": SHEF derived data created by SMToShefTransformer");
startMessageLine(sb).append(": TRACEID = ");
report.getWmoHeader();
sb.append(report.getWmoHeader());
sb.append(ccc);
sb.append(METAR_2_SHEF_NNN);
if (stnId.length() == 4) {
sb.append(stnId.substring(1));
} else if (stnId.length() == 3) {
sb.append(stnId);
}
startMessageLine(sb);
startMessageLine(sb).append(
": SHEF derived data created by SMToShefTransformer");
startMessageLine(sb).append(": TRACEID = ");
report.getWmoHeader();
sb.append(report.getWmoHeader());
String shef = closeWMOMessage(encodeShef(sb, report, headers))
.toString();
@ -256,4 +303,11 @@ public class SMToShefTransformer extends AbstractShefTransformer<ObsCommon> {
return encode;
}
}
public static void setMatchMap(
final Map<Integer, SynopticToShefRun> reportMatchMap) {
if (matchMap.size() > MAX_LIST) {
matchMap.clear();
}
matchMap.putAll(reportMatchMap);
}
}

162
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefFilter.java Normal file
View file

@ -0,0 +1,162 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.transform.shef;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.raytheon.uf.common.dataplugin.PluginDataObject;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractObsFilter;
import com.raytheon.uf.edex.decodertools.core.filterimpl.PluginDataObjectFilter;
/**
* Used to filter synoptic messages before sending them to
* {@link SMToShefTransformer}.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 28, 2015 4783 bkowal Initial creation
*
* </pre>
*
* @author bkowal
* @version 1.0
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class SynopticToShefFilter extends AbstractShefFilter {
@XmlElement
@DynamicSerializeElement
private List<SynopticToShefRun> synopticToShefRun;
/**
* Empty constructor for {@link DynamicSerialize}.
*/
public SynopticToShefFilter() {
this.synopticToShefRun = new ArrayList<>();
}
public SynopticToShefFilter(String configFile, String localContext) {
super(configFile, localContext, SynopticToShefFilter.class);
for (SynopticToShefRun run : synopticToShefRun) {
logger.info("Filter name = " + run.getFilterName()
+ " with config file: " + run.getConfigFileName());
}
}
@Override
public void init() {
this.synopticToShefRun = new ArrayList<>();
}
/*
* (non-Javadoc)
*
* @see
* com.raytheon.edex.transform.shef.AbstractShefFilter#buildRun(java.lang
* .Object, java.lang.String)
*/
@Override
protected void buildRun(Object obj, String configFile) {
if (obj instanceof PluginDataObjectFilter) {
logger.debug("Found " + configFile + " is PluginDataObjectFilter");
PluginDataObjectFilter pdof = (PluginDataObjectFilter) obj;
SynopticToShefRun run = new SynopticToShefRun();
run.setConfigFileName(null);
run.setFilterElements(pdof.getFilterElements());
run.setFilterName(pdof.getFilterName());
this.synopticToShefRun.add(run);
} else if (obj instanceof SynopticToShefFilter) {
logger.debug("Found " + configFile + " is SynopticToShefFilter");
SynopticToShefFilter filter = (SynopticToShefFilter) obj;
this.synopticToShefRun = filter.synopticToShefRun;
}
}
@Override
protected void createDummyFilter() {
SynopticToShefRun run = new SynopticToShefRun();
run.setConfigFileName(null);
// Add a dummy element.
AbstractFilterElement dummy = new AbstractFilterElement() {
@Override
public PluginDataObject filter(PluginDataObject report) {
return report;
}
};
dummy.setFilterType(AbstractObsFilter.INCLUDE_TYPE);
run.getFilterElements().add(dummy);
run.setFilterName(DUMMY_FILTER_NAME);
this.synopticToShefRun.add(run);
}
@Override
public PluginDataObject[] filter(PluginDataObject[] reports) {
Map<Integer, SynopticToShefRun> matchList = new HashMap<>();
List<PluginDataObject> reportList = new ArrayList<PluginDataObject>();
for (PluginDataObject report : reports) {
for (SynopticToShefRun run : this.synopticToShefRun) {
PluginDataObject resultRpt = filterARun(report,
run.getFilterElements());
if (resultRpt != null) {
reportList.add(resultRpt);
matchList.put(resultRpt.getId(), run);
break;
}
}
}
if (matchList.isEmpty() == false) {
SMToShefTransformer.setMatchMap(matchList);
}
return (PluginDataObject[]) reportList.toArray(new PluginDataObject[0]);
}
/**
* @return the synopticToShefRun
*/
public List<SynopticToShefRun> getSynopticToShefRun() {
return synopticToShefRun;
}
/**
* @param synopticToShefRun
* the synopticToShefRun to set
*/
public void setSynopticToShefRun(List<SynopticToShefRun> synopticToShefRun) {
this.synopticToShefRun = synopticToShefRun;
}
}

112
edexOsgi/com.raytheon.edex.plugin.shef/src/com/raytheon/edex/transform/shef/SynopticToShefRun.java Normal file
View file

@ -0,0 +1,112 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.edex.transform.shef;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import com.raytheon.uf.common.serialization.annotations.DynamicSerialize;
import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement;
import com.raytheon.uf.edex.decodertools.core.filterimpl.AbstractFilterElement;
/**
* Used to group {@link AbstractFilterElement}s with a single configuration
* file. Multiple {@link SynopticToShefRun}s can be processed by the system.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Oct 28, 2015 4783 bkowal Initial creation
*
* </pre>
*
* @author bkowal
* @version 1.0
*/
@XmlAccessorType(XmlAccessType.NONE)
@DynamicSerialize
public class SynopticToShefRun {
@XmlElement
@DynamicSerializeElement
private String configFileName;
@XmlElement
@DynamicSerializeElement
protected List<AbstractFilterElement> filterElements = new ArrayList<AbstractFilterElement>();
@XmlElement
@DynamicSerializeElement
private String filterName;
public SynopticToShefRun() {
}
/**
* @return the configFileName
*/
public String getConfigFileName() {
return configFileName;
}
/**
* @param configFileName
* the configFileName to set
*/
public void setConfigFileName(String configFileName) {
this.configFileName = configFileName;
}
/**
* @return the filterElements
*/
public List<AbstractFilterElement> getFilterElements() {
return filterElements;
}
/**
* @param filterElements
* the filterElements to set
*/
public void setFilterElements(List<AbstractFilterElement> filterElements) {
this.filterElements = filterElements;
}
/**
* @return the filterName
*/
public String getFilterName() {
return filterName;
}
/**
* @param filterName
* the filterName to set
*/
public void setFilterName(String filterName) {
this.filterName = filterName;
}
}