xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge "Issue #2905 added BUFR parsing utilities" into development

Browse source 
Former-commit-id: f19f9e2be7 [formerly 5f3ea781f5] [formerly d25c5118a3 [formerly 15f4b09539ac99bfa9d812e64deca2f3af202501]]
Former-commit-id: d25c5118a3
Former-commit-id: f4b86bd592
This commit is contained in:
Richard Peter 2014-04-04 14:42:10 -05:00 • committed by Gerrit Code Review
commit 90746fe907
8 changed files with 614 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
edexOsgi/com.raytheon.uf.common.nc.bufr/META-INF/MANIFEST.MF
View file

@ -8,9 +8,12 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Require-Bundle: ucar.nc2,
ucar.nc2.bufrsplitter,
com.raytheon.uf.common.status,
com.raytheon.uf.common.numeric
com.raytheon.uf.common.numeric,
com.raytheon.uf.common.serialization,
com.raytheon.uf.common.util
Export-Package: com.raytheon.uf.common.nc.bufr,
com.raytheon.uf.common.nc.bufr.tables,
com.raytheon.uf.common.nc.bufr.time,
com.raytheon.uf.common.nc.bufr.util
Bundle-ClassPath: .,
res/

123
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/ParsedTableUnit.java Normal file
View file

@ -0,0 +1,123 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.tables;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.raytheon.uf.common.nc.bufr.tables.TranslationTable.TableType;
/**
* Parsed representation of unit value for fields that are code or flag table
* mappings.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 27, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class ParsedTableUnit {
public static final Pattern TABLE_UNIT_PATTERN = Pattern
.compile("^\\s*(\\S+)\\s+(\\d+)-(\\d+)-(\\d+).*$");
public static final String TABLE_ID_FORMAT = "%d %02d %03d";
public static final String CODE_TABLE_TYPE = "CodeTable";
public static final String FLAG_TABLE_TYPE = "FlagTable";
private final TableType type;
private final String tableId;
/**
* @param type
* @param tableId
*/
public ParsedTableUnit(TableType type, String tableId) {
this.type = type;
this.tableId = tableId;
}
/**
* Parse unit string from BUFR file into new object
*
* @param tableUnitString
* @return
* @throws IllegalArgumentException
*/
public static ParsedTableUnit parse(String tableUnitString)
throws IllegalArgumentException {
Matcher m = TABLE_UNIT_PATTERN.matcher(tableUnitString);
if (!m.matches()) {
throw new IllegalArgumentException(
"Invalid BUFR table unit string: " + tableUnitString);
}
TableType type = matchType(m.group(1));
int f = Integer.valueOf(m.group(2));
int x = Integer.valueOf(m.group(3));
int y = Integer.valueOf(m.group(4));
return new ParsedTableUnit(type, String.format(TABLE_ID_FORMAT, f, x, y));
}
/**
* @param typeStr
* @return type of BUFR table
* @throws IllegalArgumentException
* if table type is unknown
*/
private static TableType matchType(String typeStr)
throws IllegalArgumentException {
TableType rval;
if (typeStr.equalsIgnoreCase(CODE_TABLE_TYPE)) {
rval = TableType.CODE;
} else if (typeStr.equalsIgnoreCase(FLAG_TABLE_TYPE)) {
rval = TableType.FLAG;
} else {
throw new IllegalArgumentException("Unknown BUFR table type: "
+ typeStr);
}
return rval;
}
/**
* @return the type
*/
public TableType getType() {
return type;
}
/**
* @return the tableId
*/
public String getTableId() {
return tableId;
}
}

192
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/TranslationTable.java
View file

@ -19,8 +19,15 @@
**/
package com.raytheon.uf.common.nc.bufr.tables;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -38,6 +45,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 24, 2014 2905 bclement Initial creation
* Apr 03, 2014 2905 bclement added lookup methods
*
* </pre>
*
@ -48,6 +56,10 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
public class TranslationTable {
public static enum TableType {
CODE, FLAG;
}
@XmlAttribute(required = true)
private String bufrTable;
@ -57,6 +69,16 @@ public class TranslationTable {
@XmlElement(name = "entry")
private List<TableEntry> entries;
public static final Pattern ALL_PATTERN = Pattern
.compile("^[Aa]ll\\s+(\\d+)$");
public static final Pattern RANGE_PATTERN = Pattern
.compile("^(\\d+)-(\\d+)$");
public static final Pattern SINGLE_PATTERN = Pattern.compile("^\\d+$");
private transient volatile ArrayList<TableEntry> indexedEntries;
/**
*
*/
@ -88,6 +110,161 @@ public class TranslationTable {
this.description = description;
}
/**
* Lookup table entries for given key. Table will be interpreted differently
* depending on the table type. Code tables are a one-to-one mapping of
* index to value. Flag tables are bitmaps where values are returned for
* every set bit in the key.
*
* @param key
* @param type
* @return
*/
public List<TableEntry> lookupEntries(int key, TableType type) {
if (TableType.CODE.equals(type)) {
return lookupCodeEntries(key);
} else if (TableType.FLAG.equals(type)) {
return lookupFlagEntries(key);
} else {
throw new InvalidParameterException("Invalid table type: " + type);
}
}
/**
* Get table entry at key index
*
* @param key
* @return
*/
protected List<TableEntry> lookupCodeEntries(int key) {
ArrayList<TableEntry> indexed = getIndexedEntries();
if (key < 0 || key >= indexed.size()) {
return Collections.emptyList();
}
return Arrays.asList(indexed.get(key));
}
/**
* Flag table keys are bit maps stored in an integer. The bits are aligned
* so that the first bit in the bitmap is at the most significant bit of the
* integer. This method walks the bitmap in the key and returns every table
* entry that has a set bit in the bitmap. However, if the last bit in the
* bitmap is set, only a special table entry for Missing Value is returned.
*
* @param key
* unsigned bitmap
* @return
*/
protected List<TableEntry> lookupFlagEntries(int key) {
ArrayList<TableEntry> indexed = getIndexedEntries();
final int lastIndex = indexed.size() - 1;
/* the set bit starts at 1, we want to move size - 1 bits over */
int mask = (0x80000000 >>> lastIndex);
List<TableEntry> rval;
if ((key & mask) != 0) {
/* last entry in table is Missing Value, return entry at that index */
rval = Arrays.asList(indexed.get(lastIndex));
} else {
/* not size of total table because missing value won't be included */
rval = new ArrayList<TableEntry>(lastIndex);
/* move back one from the missing value bit */
mask <<= 1;
/* walk table backwards adding included values */
for (int i = lastIndex - 1; i > -1; --i, mask <<= 1) {
if ((key & mask) != 0) {
rval.add(indexed.get(i));
}
}
}
return rval;
}
/**
* Get processed entries ordered by index. Results are cached.
*
* @return
*/
protected ArrayList<TableEntry> getIndexedEntries() {
if (entries == null) {
return new ArrayList<TableEntry>(0);
}
if (indexedEntries == null) {
synchronized (this) {
if (indexedEntries == null) {
indexedEntries = createCodeList(entries);
}
}
}
return indexedEntries;
}
/*
* priority queue node for ordered table entries
*/
private static class QueueNode {
public final int index;
public final TableEntry entry;
public static final Comparator<QueueNode> comp = new Comparator<QueueNode>() {
@Override
public int compare(QueueNode o1, QueueNode o2) {
return Integer.compare(o1.index, o2.index);
}
};
public QueueNode(int index, TableEntry entry) {
this.index = index;
this.entry = entry;
}
}
/**
* Process table entries by ordering entries by index and placing them in an
* array-backed list for easy indexing
*
* @param entries
* @return
*/
private ArrayList<TableEntry> createCodeList(List<TableEntry> entries) {
/* put in heap first to fix any order issues in the XML */
PriorityQueue<QueueNode> queue = new PriorityQueue<QueueNode>(
entries.size(), QueueNode.comp);
for (TableEntry entry : entries) {
String indexRange = entry.getIndexRange().trim();
Matcher m = SINGLE_PATTERN.matcher(indexRange);
if (m.matches()) {
int index = Integer.valueOf(indexRange);
queue.add(new QueueNode(index, entry));
} else if ((m = RANGE_PATTERN.matcher(indexRange)).matches()) {
int start = Integer.parseInt(m.group(1));
int end = Integer.parseInt(m.group(2));
for (int i = start; i <= end; ++i) {
queue.add(new QueueNode(i, entry));
}
} else if (((m = ALL_PATTERN.matcher(indexRange)).matches())) {
int index = Integer.valueOf(m.group(1));
queue.add(new QueueNode(index, entry));
} else {
throw new IllegalStateException("Invalid index range value: "
+ indexRange);
}
}
/* populate lookup table and verify table integrity */
ArrayList<TableEntry> rval = new ArrayList<TableEntry>(queue.size());
int prev = -1;
for (QueueNode node : queue) {
if (prev > -1 && prev != (node.index - 1)) {
throw new IllegalStateException("Invalid lookup table "
+ bufrTable
+ ", table must have contiguous index ranges");
}
rval.add(node.entry);
prev = node.index;
}
return rval;
}
/**
* Append new entry to entries
*
@ -115,6 +292,21 @@ public class TranslationTable {
this.bufrTable = bufrTable;
}
/**
* @return the description
*/
public String getDescription() {
return description;
}
/**
* @param description
* the description to set
*/
public void setDescription(String description) {
this.description = description;
}
/**
* @return the entries
*/

59
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/TranslationTableManager.java Normal file
View file

@ -0,0 +1,59 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.tables;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.serialization.JAXBManager;
/**
* JAXB manager for translation table objects
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 31, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class TranslationTableManager extends JAXBManager {
private static volatile TranslationTableManager instance;
private TranslationTableManager() throws JAXBException {
super(TranslationTable.class);
}
public static TranslationTableManager getInstance() throws JAXBException {
if (instance == null) {
synchronized (TranslationTableManager.class) {
instance = new TranslationTableManager();
}
}
return instance;
}
}

120
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/time/BufrTimeFieldParser.java Normal file
View file

@ -0,0 +1,120 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.time;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.DatatypeConverter;
/**
* Parser utility for NetCDF BUFR time fields
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 28, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class BufrTimeFieldParser {
public static final String SECONDS_OFFSET = "seconds";
public static final String MINUTES_OFFSET = "minutes";
public static final String HOURS_OFFSET = "hours";
public static final String DAYS_OFFSET = "days";
public static final Map<String, Integer> CAL_FIELD_MAP;
public static final Pattern TIME_UNIT_PATTERN = Pattern
.compile("^\\s*(\\S+)\\s+since\\s+(\\S+)\\s*$");
static {
Map<String, Integer> map = new HashMap<String, Integer>();
map.put(SECONDS_OFFSET, Calendar.SECOND);
map.put(MINUTES_OFFSET, Calendar.MINUTE);
map.put(HOURS_OFFSET, Calendar.HOUR_OF_DAY);
map.put(DAYS_OFFSET, Calendar.DAY_OF_MONTH);
CAL_FIELD_MAP = Collections.unmodifiableMap(map);
}
/**
*
*/
private BufrTimeFieldParser() {
}
/**
* Parse units for time reference and add offset value. Results returned in
* calendar object.
*
* @param value
* @param fieldUnits
* @return
* @throws TimeFieldParseException
*/
public static Calendar processTimeField(Object value, String fieldUnits)
throws TimeFieldParseException {
if (value == null) {
return null;
}
if (!(value instanceof Number)) {
throw new TimeFieldParseException(
"Time field with non-numeric value: " + value);
}
Number offset = (Number) value;
Matcher m = TIME_UNIT_PATTERN.matcher(fieldUnits);
if (m.matches()) {
String offsetUnits = m.group(1).trim();
Integer calField = CAL_FIELD_MAP.get(offsetUnits);
if (calField == null) {
throw new TimeFieldParseException(
"Unsupported time offset unit: " + offsetUnits);
}
String refString = m.group(2).trim();
Calendar refCal;
try {
refCal = DatatypeConverter.parseDateTime(refString);
} catch (Exception e) {
throw new TimeFieldParseException(
"Unsupported time reference string format: "
+ refString);
}
refCal.add(calField, offset.intValue());
return refCal;
} else {
throw new TimeFieldParseException(
"Time field with unknown unit format: " + fieldUnits);
}
}
}

58
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/time/TimeFieldParseException.java Normal file
View file

@ -0,0 +1,58 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.time;
/**
* Exception thrown during NetCDF BUFR time processing
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 28, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class TimeFieldParseException extends Exception {
private static final long serialVersionUID = -2830364208081317852L;
public TimeFieldParseException() {
super();
}
public TimeFieldParseException(String message, Throwable cause) {
super(message, cause);
}
public TimeFieldParseException(String message) {
super(message);
}
public TimeFieldParseException(Throwable cause) {
super(cause);
}
}

56
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/util/BufrMapper.java Normal file
View file

@ -0,0 +1,56 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.util;
import java.io.File;
import java.util.Collection;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.util.mapping.Mapper;
/**
* Mapping for BUFR field aliases to point data container parameters
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 27, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class BufrMapper extends Mapper {
/**
* @throws JAXBException
*/
public BufrMapper(Collection<File> aliasFiles) throws JAXBException {
for (File aliasFile : aliasFiles) {
addAliasList(aliasFile);
}
}
}

3
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/util/TranslationTableGenerator.java
View file

@ -48,6 +48,7 @@ import com.raytheon.uf.common.status.UFStatus;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 24, 2014 2905 bclement Initial creation
* Apr 02, 2014 2905 bclement made replaceWhiteSpace() public
*
* </pre>
*
@ -153,7 +154,7 @@ public class TranslationTableGenerator {
* @param replacement
* @return
*/
private static String replaceWhiteSpace(String str, String replacement) {
public static String replaceWhiteSpace(String str, String replacement) {
Matcher m = WmoCodeTableParser.WHITESPACE.matcher(str);
return m.replaceAll(replacement);
}