xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #2905 added BUFR parsing utilities

Browse source 
added table units parser for BUFR table values
added time units parser for BUFR time values
added lookup methods to translation table
added mapping class for BUFR fields to point data container parameters


Former-commit-id: 88c338e999 [formerly 43512fd90a] [formerly ed7a808b87] [formerly dacfc4082d [formerly ed7a808b87 [formerly eaeab6dc6dfc9e0f3544ccb1ebea53669d2222c7]]]
Former-commit-id: dacfc4082d
Former-commit-id: 20ed830bae87a08693d57dfd87d5239335e3b47e [formerly 1ff0baa6b7]
Former-commit-id: 261dccba3c
This commit is contained in:
Brian Clements 2014-04-03 16:23:55 -05:00
parent 7138f246cc
commit 35b686c0ba
8 changed files with 614 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
edexOsgi/com.raytheon.uf.common.nc.bufr/META-INF/MANIFEST.MF
View file

@ -8,9 +8,12 @@ Bundle-RequiredExecutionEnvironment: JavaSE-1.6
Require-Bundle: ucar.nc2,
ucar.nc2.bufrsplitter,
com.raytheon.uf.common.status,
com.raytheon.uf.common.numeric
com.raytheon.uf.common.numeric,
com.raytheon.uf.common.serialization,
com.raytheon.uf.common.util
Export-Package: com.raytheon.uf.common.nc.bufr,
com.raytheon.uf.common.nc.bufr.tables,
com.raytheon.uf.common.nc.bufr.time,
com.raytheon.uf.common.nc.bufr.util
Bundle-ClassPath: .,
res/

123
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/ParsedTableUnit.java Normal file
View file

@ -0,0 +1,123 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.tables;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.raytheon.uf.common.nc.bufr.tables.TranslationTable.TableType;
/**
* Parsed representation of unit value for fields that are code or flag table
* mappings.
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 27, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class ParsedTableUnit {
public static final Pattern TABLE_UNIT_PATTERN = Pattern
.compile("^\\s*(\\S+)\\s+(\\d+)-(\\d+)-(\\d+).*$");
public static final String TABLE_ID_FORMAT = "%d %02d %03d";
public static final String CODE_TABLE_TYPE = "CodeTable";
public static final String FLAG_TABLE_TYPE = "FlagTable";
private final TableType type;
private final String tableId;
/**
* @param type
* @param tableId
*/
public ParsedTableUnit(TableType type, String tableId) {
this.type = type;
this.tableId = tableId;
}
/**
* Parse unit string from BUFR file into new object
*
* @param tableUnitString
* @return
* @throws IllegalArgumentException
*/
public static ParsedTableUnit parse(String tableUnitString)
throws IllegalArgumentException {
Matcher m = TABLE_UNIT_PATTERN.matcher(tableUnitString);
if (!m.matches()) {
throw new IllegalArgumentException(
"Invalid BUFR table unit string: " + tableUnitString);
}
TableType type = matchType(m.group(1));
int f = Integer.valueOf(m.group(2));
int x = Integer.valueOf(m.group(3));
int y = Integer.valueOf(m.group(4));
return new ParsedTableUnit(type, String.format(TABLE_ID_FORMAT, f, x, y));
}
/**
* @param typeStr
* @return type of BUFR table
* @throws IllegalArgumentException
* if table type is unknown
*/
private static TableType matchType(String typeStr)
throws IllegalArgumentException {
TableType rval;
if (typeStr.equalsIgnoreCase(CODE_TABLE_TYPE)) {
rval = TableType.CODE;
} else if (typeStr.equalsIgnoreCase(FLAG_TABLE_TYPE)) {
rval = TableType.FLAG;
} else {
throw new IllegalArgumentException("Unknown BUFR table type: "
+ typeStr);
}
return rval;
}
/**
* @return the type
*/
public TableType getType() {
return type;
}
/**
* @return the tableId
*/
public String getTableId() {
return tableId;
}
}

192
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/TranslationTable.java
View file

@ -19,8 +19,15 @@
**/
package com.raytheon.uf.common.nc.bufr.tables;
import java.security.InvalidParameterException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -38,6 +45,7 @@ import javax.xml.bind.annotation.XmlRootElement;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 24, 2014 2905 bclement Initial creation
* Apr 03, 2014 2905 bclement added lookup methods
*
* </pre>
*
@ -48,6 +56,10 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
public class TranslationTable {
public static enum TableType {
CODE, FLAG;
}
@XmlAttribute(required = true)
private String bufrTable;
@ -57,6 +69,16 @@ public class TranslationTable {
@XmlElement(name = "entry")
private List<TableEntry> entries;
public static final Pattern ALL_PATTERN = Pattern
.compile("^[Aa]ll\\s+(\\d+)$");
public static final Pattern RANGE_PATTERN = Pattern
.compile("^(\\d+)-(\\d+)$");
public static final Pattern SINGLE_PATTERN = Pattern.compile("^\\d+$");
private transient volatile ArrayList<TableEntry> indexedEntries;
/**
*
*/
@ -88,6 +110,161 @@ public class TranslationTable {
this.description = description;
}
/**
* Lookup table entries for given key. Table will be interpreted differently
* depending on the table type. Code tables are a one-to-one mapping of
* index to value. Flag tables are bitmaps where values are returned for
* every set bit in the key.
*
* @param key
* @param type
* @return
*/
public List<TableEntry> lookupEntries(int key, TableType type) {
if (TableType.CODE.equals(type)) {
return lookupCodeEntries(key);
} else if (TableType.FLAG.equals(type)) {
return lookupFlagEntries(key);
} else {
throw new InvalidParameterException("Invalid table type: " + type);
}
}
/**
* Get table entry at key index
*
* @param key
* @return
*/
protected List<TableEntry> lookupCodeEntries(int key) {
ArrayList<TableEntry> indexed = getIndexedEntries();
if (key < 0 || key >= indexed.size()) {
return Collections.emptyList();
}
return Arrays.asList(indexed.get(key));
}
/**
* Flag table keys are bit maps stored in an integer. The bits are aligned
* so that the first bit in the bitmap is at the most significant bit of the
* integer. This method walks the bitmap in the key and returns every table
* entry that has a set bit in the bitmap. However, if the last bit in the
* bitmap is set, only a special table entry for Missing Value is returned.
*
* @param key
* unsigned bitmap
* @return
*/
protected List<TableEntry> lookupFlagEntries(int key) {
ArrayList<TableEntry> indexed = getIndexedEntries();
final int lastIndex = indexed.size() - 1;
/* the set bit starts at 1, we want to move size - 1 bits over */
int mask = (0x80000000 >>> lastIndex);
List<TableEntry> rval;
if ((key & mask) != 0) {
/* last entry in table is Missing Value, return entry at that index */
rval = Arrays.asList(indexed.get(lastIndex));
} else {
/* not size of total table because missing value won't be included */
rval = new ArrayList<TableEntry>(lastIndex);
/* move back one from the missing value bit */
mask <<= 1;
/* walk table backwards adding included values */
for (int i = lastIndex - 1; i > -1; --i, mask <<= 1) {
if ((key & mask) != 0) {
rval.add(indexed.get(i));
}
}
}
return rval;
}
/**
* Get processed entries ordered by index. Results are cached.
*
* @return
*/
protected ArrayList<TableEntry> getIndexedEntries() {
if (entries == null) {
return new ArrayList<TableEntry>(0);
}
if (indexedEntries == null) {
synchronized (this) {
if (indexedEntries == null) {
indexedEntries = createCodeList(entries);
}
}
}
return indexedEntries;
}
/*
* priority queue node for ordered table entries
*/
private static class QueueNode {
public final int index;
public final TableEntry entry;
public static final Comparator<QueueNode> comp = new Comparator<QueueNode>() {
@Override
public int compare(QueueNode o1, QueueNode o2) {
return Integer.compare(o1.index, o2.index);
}
};
public QueueNode(int index, TableEntry entry) {
this.index = index;
this.entry = entry;
}
}
/**
* Process table entries by ordering entries by index and placing them in an
* array-backed list for easy indexing
*
* @param entries
* @return
*/
private ArrayList<TableEntry> createCodeList(List<TableEntry> entries) {
/* put in heap first to fix any order issues in the XML */
PriorityQueue<QueueNode> queue = new PriorityQueue<QueueNode>(
entries.size(), QueueNode.comp);
for (TableEntry entry : entries) {
String indexRange = entry.getIndexRange().trim();
Matcher m = SINGLE_PATTERN.matcher(indexRange);
if (m.matches()) {
int index = Integer.valueOf(indexRange);
queue.add(new QueueNode(index, entry));
} else if ((m = RANGE_PATTERN.matcher(indexRange)).matches()) {
int start = Integer.parseInt(m.group(1));
int end = Integer.parseInt(m.group(2));
for (int i = start; i <= end; ++i) {
queue.add(new QueueNode(i, entry));
}
} else if (((m = ALL_PATTERN.matcher(indexRange)).matches())) {
int index = Integer.valueOf(m.group(1));
queue.add(new QueueNode(index, entry));
} else {
throw new IllegalStateException("Invalid index range value: "
+ indexRange);
}
}
/* populate lookup table and verify table integrity */
ArrayList<TableEntry> rval = new ArrayList<TableEntry>(queue.size());
int prev = -1;
for (QueueNode node : queue) {
if (prev > -1 && prev != (node.index - 1)) {
throw new IllegalStateException("Invalid lookup table "
+ bufrTable
+ ", table must have contiguous index ranges");
}
rval.add(node.entry);
prev = node.index;
}
return rval;
}
/**
* Append new entry to entries
*
@ -115,6 +292,21 @@ public class TranslationTable {
this.bufrTable = bufrTable;
}
/**
* @return the description
*/
public String getDescription() {
return description;
}
/**
* @param description
* the description to set
*/
public void setDescription(String description) {
this.description = description;
}
/**
* @return the entries
*/

59
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/tables/TranslationTableManager.java Normal file
View file

@ -0,0 +1,59 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.tables;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.serialization.JAXBManager;
/**
* JAXB manager for translation table objects
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 31, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class TranslationTableManager extends JAXBManager {
private static volatile TranslationTableManager instance;
private TranslationTableManager() throws JAXBException {
super(TranslationTable.class);
}
public static TranslationTableManager getInstance() throws JAXBException {
if (instance == null) {
synchronized (TranslationTableManager.class) {
instance = new TranslationTableManager();
}
}
return instance;
}
}

120
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/time/BufrTimeFieldParser.java Normal file
View file

@ -0,0 +1,120 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.time;
import java.util.Calendar;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.DatatypeConverter;
/**
* Parser utility for NetCDF BUFR time fields
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 28, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class BufrTimeFieldParser {
public static final String SECONDS_OFFSET = "seconds";
public static final String MINUTES_OFFSET = "minutes";
public static final String HOURS_OFFSET = "hours";
public static final String DAYS_OFFSET = "days";
public static final Map<String, Integer> CAL_FIELD_MAP;
public static final Pattern TIME_UNIT_PATTERN = Pattern
.compile("^\\s*(\\S+)\\s+since\\s+(\\S+)\\s*$");
static {
Map<String, Integer> map = new HashMap<String, Integer>();
map.put(SECONDS_OFFSET, Calendar.SECOND);
map.put(MINUTES_OFFSET, Calendar.MINUTE);
map.put(HOURS_OFFSET, Calendar.HOUR_OF_DAY);
map.put(DAYS_OFFSET, Calendar.DAY_OF_MONTH);
CAL_FIELD_MAP = Collections.unmodifiableMap(map);
}
/**
*
*/
private BufrTimeFieldParser() {
}
/**
* Parse units for time reference and add offset value. Results returned in
* calendar object.
*
* @param value
* @param fieldUnits
* @return
* @throws TimeFieldParseException
*/
public static Calendar processTimeField(Object value, String fieldUnits)
throws TimeFieldParseException {
if (value == null) {
return null;
}
if (!(value instanceof Number)) {
throw new TimeFieldParseException(
"Time field with non-numeric value: " + value);
}
Number offset = (Number) value;
Matcher m = TIME_UNIT_PATTERN.matcher(fieldUnits);
if (m.matches()) {
String offsetUnits = m.group(1).trim();
Integer calField = CAL_FIELD_MAP.get(offsetUnits);
if (calField == null) {
throw new TimeFieldParseException(
"Unsupported time offset unit: " + offsetUnits);
}
String refString = m.group(2).trim();
Calendar refCal;
try {
refCal = DatatypeConverter.parseDateTime(refString);
} catch (Exception e) {
throw new TimeFieldParseException(
"Unsupported time reference string format: "
+ refString);
}
refCal.add(calField, offset.intValue());
return refCal;
} else {
throw new TimeFieldParseException(
"Time field with unknown unit format: " + fieldUnits);
}
}
}

58
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/time/TimeFieldParseException.java Normal file
View file

@ -0,0 +1,58 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.time;
/**
* Exception thrown during NetCDF BUFR time processing
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 28, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class TimeFieldParseException extends Exception {
private static final long serialVersionUID = -2830364208081317852L;
public TimeFieldParseException() {
super();
}
public TimeFieldParseException(String message, Throwable cause) {
super(message, cause);
}
public TimeFieldParseException(String message) {
super(message);
}
public TimeFieldParseException(Throwable cause) {
super(cause);
}
}

56
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/util/BufrMapper.java Normal file
View file

@ -0,0 +1,56 @@
/**
* This software was developed and / or modified by Raytheon Company,
* pursuant to Contract DG133W-05-CQ-1067 with the US Government.
*
* U.S. EXPORT CONTROLLED TECHNICAL DATA
* This software product contains export-restricted data whose
* export/transfer/disclosure is restricted by U.S. law. Dissemination
* to non-U.S. persons whether in the United States or abroad requires
* an export license or other authorization.
*
* Contractor Name: Raytheon Company
* Contractor Address: 6825 Pine Street, Suite 340
* Mail Stop B8
* Omaha, NE 68106
* 402.291.0100
*
* See the AWIPS II Master Rights File ("Master Rights File.pdf") for
* further licensing information.
**/
package com.raytheon.uf.common.nc.bufr.util;
import java.io.File;
import java.util.Collection;
import javax.xml.bind.JAXBException;
import com.raytheon.uf.common.util.mapping.Mapper;
/**
* Mapping for BUFR field aliases to point data container parameters
*
* <pre>
*
* SOFTWARE HISTORY
*
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 27, 2014 2905 bclement Initial creation
*
* </pre>
*
* @author bclement
* @version 1.0
*/
public class BufrMapper extends Mapper {
/**
* @throws JAXBException
*/
public BufrMapper(Collection<File> aliasFiles) throws JAXBException {
for (File aliasFile : aliasFiles) {
addAliasList(aliasFile);
}
}
}

3
edexOsgi/com.raytheon.uf.common.nc.bufr/src/com/raytheon/uf/common/nc/bufr/util/TranslationTableGenerator.java
View file

@ -48,6 +48,7 @@ import com.raytheon.uf.common.status.UFStatus;
* Date Ticket# Engineer Description
* ------------ ---------- ----------- --------------------------
* Mar 24, 2014 2905 bclement Initial creation
* Apr 02, 2014 2905 bclement made replaceWhiteSpace() public
*
* </pre>
*
@ -153,7 +154,7 @@ public class TranslationTableGenerator {
* @param replacement
* @return
*/
private static String replaceWhiteSpace(String str, String replacement) {
public static String replaceWhiteSpace(String str, String replacement) {
Matcher m = WmoCodeTableParser.WHITESPACE.matcher(str);
return m.replaceAll(replacement);
}