headers) {
+ Object gridPointsHeader = headers.get(GRID_POINT_COUNT_HEADER);
+ if (gridPointsHeader instanceof Number) {
+ long gridPoints = ((Number) gridPointsHeader).longValue();
+ release(gridPoints);
+ }
+ }
+
+ private void reserve(long gridPoints) {
+ if (!fastReserve(gridPoints, false)) {
+ ITimer timer = TimeUtil.getTimer();
+ timer.start();
+ Object waiter = new Object();
+ synchronized (waiter) {
+ waiters.offer(waiter);
+ while (waiters.peek() != waiter
+ || !fastReserve(gridPoints, true)) {
+ try {
+ waiter.wait(15000);
+ } catch (InterruptedException e) {
+ ;
+ }
+ }
+ }
+ waiters.remove(waiter);
+ notifyNext();
+ timer.stop();
+ perfLog.logDuration("Grib: Time waiting to reserve grid points",
+ timer.getElapsedTime());
+ }
+ }
+
+ private boolean fastReserve(long gridPoints, boolean ignoreWaiters) {
+ long oldPoints = currentPoints.get();
+ long newPoints = oldPoints + gridPoints;
+ while ((ignoreWaiters || waiters.isEmpty())
+ && newPoints <= maxConcurrentGridPoints) {
+ if (currentPoints.compareAndSet(oldPoints, newPoints)) {
+ return true;
+ }
+ oldPoints = this.currentPoints.get();
+ newPoints = oldPoints + gridPoints;
+ }
+ return false;
+ }
+
+ private void release(long gridPoints) {
+ currentPoints.addAndGet(-gridPoints);
+ notifyNext();
+ }
+
+ private void notifyNext() {
+ Object next = waiters.peek();
+ if (next != null) {
+ synchronized (next) {
+ next.notifyAll();
+ }
+ }
+ }
+
+}
diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLargeFileChecker.java b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLargeFileChecker.java
deleted file mode 100644
index 9c92747674..0000000000
--- a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLargeFileChecker.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * This software was developed and / or modified by Raytheon Company,
- * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
- *
- * U.S. EXPORT CONTROLLED TECHNICAL DATA
- * This software product contains export-restricted data whose
- * export/transfer/disclosure is restricted by U.S. law. Dissemination
- * to non-U.S. persons whether in the United States or abroad requires
- * an export license or other authorization.
- *
- * Contractor Name: Raytheon Company
- * Contractor Address: 6825 Pine Street, Suite 340
- * Mail Stop B8
- * Omaha, NE 68106
- * 402.291.0100
- *
- * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
- * further licensing information.
- **/
-package com.raytheon.edex.plugin.grib;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-import org.apache.camel.Exchange;
-import org.apache.camel.Processor;
-
-import com.raytheon.edex.plugin.grib.exception.GribException;
-import com.raytheon.uf.common.localization.IPathManager;
-import com.raytheon.uf.common.localization.LocalizationContext;
-import com.raytheon.uf.common.localization.PathManagerFactory;
-import com.raytheon.uf.common.serialization.SerializationUtil;
-import com.raytheon.uf.common.status.IUFStatusHandler;
-import com.raytheon.uf.common.status.UFStatus;
-import com.raytheon.uf.common.status.UFStatus.Priority;
-import com.raytheon.uf.edex.database.cluster.ClusterLockUtils;
-import com.raytheon.uf.edex.database.cluster.ClusterLockUtils.LockState;
-import com.raytheon.uf.edex.database.cluster.ClusterTask;
-
-/**
- * Implementation of a Camel Message Processor to check if the ingested grib
- * file matches one of the designated "large" grib files. If this processor
- * detects a file containing one of WMO regexes contained in the
- * largeGribPatterns.xml, all other grib ingest threads will block to wait for
- * this file to finish processing. Cluster locking is used. The lock is obtained
- * in this Processor. A second processor, GribLockRelease, is used to release
- * the lock once the file is done being processed.
- *
- * This processor was put in place to ensure large grib files will not throw out
- * of memory exceptions while processing.
- *
- *
- *
- *
- * SOFTWARE HISTORY
- *
- * Date Ticket# Engineer Description
- * ------------- -------- ----------- --------------------------
- * Oct 15, 2010 6644 bphillip Initial Creation
- * Jul 18, 2013 2194 bsteffen Fix site override.
- * Oct 07, 2013 2042 bsteffen Decode GribDecodeMessage instead of
- * files.
- *
- *
- *
- * @author bphillip
- * @version 1
- * @see com.raytheon.edex.plugin.grib.GribLockRelease
- */
-public class GribLargeFileChecker implements Processor {
- private static final transient IUFStatusHandler statusHandler = UFStatus
- .getHandler(GribLargeFileChecker.class);
-
- /**
- * The header attribute attached to the message to denote whether this file
- * will receive single threaded handling
- */
- public static final String LARGE_FILE_HEADER = "largeFile";
-
- /** The cluster task name used for cluster locking */
- public static final String CLUSTER_TASK_NAME = "GribIngestLargeFile";
-
- public static final String CLUSTER_TASK_DETAILS;
-
- /**
- * The base localization patterns used to specify which WMO header regexes
- * receive special handling
- */
- private static LargeGribPatterns basePatterns;
-
- /**
- * The site localization patterns used to specify which WMO header regexes
- * receive special handling
- */
- private static LargeGribPatterns sitePatterns;
-
- static {
- String host = null;
- try {
- // Initialize the cluster task name with the host name
- host = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- e.printStackTrace();
- host = "";
- }
- CLUSTER_TASK_DETAILS = host;
- }
-
- @Override
- public void process(Exchange exchange) throws Exception {
- if (basePatterns == null) {
- loadPatterns();
- }
- GribDecodeMessage message = (GribDecodeMessage) exchange.getIn()
- .getBody();
- File gribFile = new File(message.getFileName());
- String header = (String) exchange.getIn().getHeader("header");
- if (header == null) {
- // No header entry so will try and use the filename instead
- header = gribFile.getName();
- }
-
- ClusterTask task = ClusterLockUtils.lookupLock(CLUSTER_TASK_NAME,
- CLUSTER_TASK_DETAILS);
- boolean waitForLargeGrib = task.isRunning();
-
- while (waitForLargeGrib) {
- Thread.sleep(500);
- task = ClusterLockUtils.lookupLock(CLUSTER_TASK_NAME,
- CLUSTER_TASK_DETAILS);
- waitForLargeGrib = task.isRunning();
-
- // need to handle timing out of large grib file process manually
- if (waitForLargeGrib
- && (System.currentTimeMillis() - task.getLastExecution()) > 120000) {
- statusHandler
- .handle(Priority.EVENTA,
- "Large Grib file process timed out. Clearing lock and resuming processing");
- ClusterLockUtils
- .unlock(CLUSTER_TASK_NAME, CLUSTER_TASK_DETAILS);
- waitForLargeGrib = false;
- }
- }
-
- if (isLarge(header)) {
- statusHandler.handle(Priority.EVENTA,
- "Large Grib file detected. Establishing lock.");
- while (!ClusterLockUtils
- .lock(CLUSTER_TASK_NAME, CLUSTER_TASK_DETAILS, 120000, true)
- .getLockState().equals(LockState.SUCCESSFUL)) {
- Thread.sleep(100);
- }
- statusHandler.handle(Priority.EVENTA,
- "Large Grib file lock established.");
- // Wait for other threads to complete processing before we proceed
- Thread.sleep(1000);
- exchange.getIn().setHeader(LARGE_FILE_HEADER, true);
-
- } else {
- exchange.getIn().setHeader(LARGE_FILE_HEADER, false);
- }
- exchange.getIn().setHeader("dequeueTime", System.currentTimeMillis());
-
- }
-
- /**
- * Loads the WMO header patterns for grib files which receive special
- * handling
- *
- * @throws GribException
- * If the patterns cannot be loaded
- */
- private void loadPatterns() throws GribException {
- IPathManager pathMgr = PathManagerFactory.getPathManager();
- LocalizationContext commonStaticBase = pathMgr.getContext(
- LocalizationContext.LocalizationType.EDEX_STATIC,
- LocalizationContext.LocalizationLevel.BASE);
-
- LocalizationContext siteStaticBase = pathMgr.getContext(
- LocalizationContext.LocalizationType.EDEX_STATIC,
- LocalizationContext.LocalizationLevel.SITE);
- String path = "";
- String sitePath = "";
- try {
- path = pathMgr.getFile(commonStaticBase,
- "grib" + File.separator + "largeGribPatterns.xml")
- .getCanonicalPath();
- sitePath = pathMgr.getFile(siteStaticBase,
- "grib" + File.separator + "largeGribPatterns.xml")
- .getCanonicalPath();
- } catch (IOException e) {
- throw new GribException("Error reading large grib patterns", e);
- }
-
- File modelFile = new File(path);
- File siteModelFile = new File(sitePath);
- if (siteModelFile.exists()) {
- sitePatterns = loadPatterns(siteModelFile);
- }
- basePatterns = loadPatterns(modelFile);
-
- }
-
- /**
- * Loads the regex patterns from the specified file
- *
- * @param modelFile
- * The file to load the regex patterns from
- * @return An object containing the compiled regex patterns
- * @throws GribException
- * If the patterns cannot be loaded
- */
- private LargeGribPatterns loadPatterns(File modelFile) throws GribException {
- LargeGribPatterns patternSet = null;
- try {
- patternSet = (LargeGribPatterns) SerializationUtil
- .jaxbUnmarshalFromXmlFile(modelFile.getPath());
- } catch (Exception e) {
- throw new GribException("File " + modelFile.getAbsolutePath()
- + " could not be unmarshalled.", e);
- }
- patternSet.compilePatterns();
- return patternSet;
- }
-
- /**
- * Checks whether the provided header matches the regexes designated for
- * special handling
- *
- * @param header
- * The header to check
- * @return True if the file associated with this WMO header is to receive
- * special handling
- */
- private boolean isLarge(String header) {
- boolean isLarge = false;
- if (sitePatterns != null) {
- isLarge = sitePatterns.isDesiredHeader(header);
- }
- if (!isLarge) {
- isLarge = basePatterns.isDesiredHeader(header);
- }
- return isLarge;
- }
-}
diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLockRelease.java b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLockRelease.java
deleted file mode 100644
index f59e2c8d02..0000000000
--- a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribLockRelease.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * This software was developed and / or modified by Raytheon Company,
- * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
- *
- * U.S. EXPORT CONTROLLED TECHNICAL DATA
- * This software product contains export-restricted data whose
- * export/transfer/disclosure is restricted by U.S. law. Dissemination
- * to non-U.S. persons whether in the United States or abroad requires
- * an export license or other authorization.
- *
- * Contractor Name: Raytheon Company
- * Contractor Address: 6825 Pine Street, Suite 340
- * Mail Stop B8
- * Omaha, NE 68106
- * 402.291.0100
- *
- * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
- * further licensing information.
- **/
-package com.raytheon.edex.plugin.grib;
-
-import org.apache.camel.Exchange;
-import org.apache.camel.Processor;
-
-import com.raytheon.uf.common.status.IUFStatusHandler;
-import com.raytheon.uf.common.status.UFStatus;
-import com.raytheon.uf.common.status.UFStatus.Priority;
-import com.raytheon.uf.edex.database.cluster.ClusterLockUtils;
-
-/**
- * Releases lock obtained in the GribLargeFileChecker class if one was
- * established.
- *
- *
- *
- * SOFTWARE HISTORY
- *
- * Date Ticket# Engineer Description
- * ------------ ---------- ----------- --------------------------
- * 10/15/10 6644 bphillip Initial Creation
- *
- *
- *
- * @author bphillip
- * @version 1
- * @see com.raytheon.edex.plugin.grib.GribLargeFileChecker
- */
-public class GribLockRelease implements Processor {
- private static final transient IUFStatusHandler statusHandler = UFStatus
- .getHandler(GribLockRelease.class);
-
- @Override
- public void process(Exchange exchange) throws Exception {
- Boolean isLargeFile = (Boolean) exchange.getIn().getHeader(
- GribLargeFileChecker.LARGE_FILE_HEADER);
- if (isLargeFile) {
- boolean success = ClusterLockUtils.unlock(
- GribLargeFileChecker.CLUSTER_TASK_NAME,
- GribLargeFileChecker.CLUSTER_TASK_DETAILS);
- if (success) {
- statusHandler.handle(Priority.EVENTA,
- "Large Grib file lock released!");
- } else {
- statusHandler.handle(Priority.CRITICAL,
- "Large Grib file lock could not be released!!");
- }
- }
- }
-}
diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribSplitter.java b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribSplitter.java
index 8368fe53b1..208a475d28 100644
--- a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribSplitter.java
+++ b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/GribSplitter.java
@@ -29,6 +29,8 @@ import ucar.unidata.io.KMPMatch;
import ucar.unidata.io.RandomAccessFile;
import com.raytheon.edex.plugin.grib.exception.GribException;
+import com.raytheon.edex.plugin.grib.spatial.GribSpatialCache;
+import com.raytheon.uf.common.gridcoverage.GridCoverage;
import com.raytheon.uf.common.status.IUFStatusHandler;
import com.raytheon.uf.common.status.UFStatus;
import com.raytheon.uf.common.status.UFStatus.Priority;
@@ -61,7 +63,7 @@ public class GribSplitter {
List messages = new ArrayList();
RandomAccessFile raf = null;
try {
- raf = new RandomAccessFile(file.getAbsolutePath(), "r", 1024);
+ raf = new RandomAccessFile(file.getAbsolutePath(), "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
while (raf.searchForward(matcher, Integer.MAX_VALUE)) {
GribDecodeMessage message = new GribDecodeMessage();
@@ -73,6 +75,22 @@ public class GribSplitter {
message.setGribEdition((byte) is.getGribEdition());
long length = is.getGribLength();
message.setMessageLength(length);
+ switch (is.getGribEdition()) {
+ case 1:
+ message.setGridPointCount(getGrib1GridPointCount(raf,
+ startPosition));
+ break;
+ case 2:
+ message.setGridPointCount(getGrib2GridPointCount(raf,
+ startPosition, length));
+ break;
+ default:
+ /* This is not a grid we can handle. Let the message proceed to the decoder to throw errors.*/
+ message.setGridPointCount(0);
+ break;
+ }
+ System.out.println("Number of points is: "
+ + message.getGridPointCount());
messages.add(message);
raf.seek(startPosition + length);
/*
@@ -99,4 +117,59 @@ public class GribSplitter {
return messages;
}
+ private long getGrib2GridPointCount(RandomAccessFile raf,
+ long startPosition, long messageLength) throws IOException {
+ long gridPointCount = 0;
+ long totalGridPointCount = 0;
+ long start = startPosition + 16;
+ while (start < startPosition + messageLength - 4) {
+ raf.seek(start);
+ int length = raf.readInt();
+ int section = raf.readByte();
+ switch (section) {
+ case 3:
+ raf.seek(start + 30);
+ int nx = raf.readInt();
+ int ny = raf.readInt();
+ gridPointCount = nx * ny;
+ break;
+ case 7:
+ totalGridPointCount += gridPointCount;
+ break;
+ }
+ start += length;
+ }
+ return totalGridPointCount;
+ }
+
+ private long getGrib1GridPointCount(RandomAccessFile raf,
+ long startPosition)
+ throws IOException {
+ raf.seek(startPosition + 8);
+ int pdsLength = (raf.readUnsignedShort() << 8) + raf.readUnsignedByte();
+ raf.skipBytes(3);
+ int grid = raf.readUnsignedByte();
+ int gdsPresent = raf.readUnsignedByte() & 0x80;
+ if (grid != 255) {
+ GridCoverage coverage = GribSpatialCache.getInstance()
+ .getGridByName(String.valueOf(grid));
+ if (coverage != null) {
+ return coverage.getNx() * coverage.getNy();
+ }
+ }
+ if (gdsPresent != 0) {
+ raf.seek(startPosition + 8 + pdsLength + 6);
+ /*
+ * Note: for Quasi-regular grids nx or ny may be coded as 0xFFFF
+ * which will result ina dramatic overestimate of the number of grid
+ * points. All such known grids are identified by grid number above.
+ */
+ int nx = raf.readUnsignedShort();
+ int ny = raf.readUnsignedShort();
+ return nx * ny;
+ }
+ /* This is not a grid we can handle. Let the message proceed to the decoder to throw errors.*/
+ return 0;
+ }
+
}
diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/LargeGribPatterns.java b/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/LargeGribPatterns.java
deleted file mode 100644
index 5659e168da..0000000000
--- a/edexOsgi/com.raytheon.edex.plugin.grib/src/com/raytheon/edex/plugin/grib/LargeGribPatterns.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * This software was developed and / or modified by Raytheon Company,
- * pursuant to Contract DG133W-05-CQ-1067 with the US Government.
- *
- * U.S. EXPORT CONTROLLED TECHNICAL DATA
- * This software product contains export-restricted data whose
- * export/transfer/disclosure is restricted by U.S. law. Dissemination
- * to non-U.S. persons whether in the United States or abroad requires
- * an export license or other authorization.
- *
- * Contractor Name: Raytheon Company
- * Contractor Address: 6825 Pine Street, Suite 340
- * Mail Stop B8
- * Omaha, NE 68106
- * 402.291.0100
- *
- * See the AWIPS II Master Rights File ("Master Rights File.pdf") for
- * further licensing information.
- **/
-package com.raytheon.edex.plugin.grib;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlElements;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import com.raytheon.uf.common.serialization.ISerializableObject;
-
-/**
- * Object containing the Grib Patterns which receive single threaded decode
- * handling.
- *
- *
- *
- * SOFTWARE HISTORY
- *
- * Date Ticket# Engineer Description
- * ------------ ---------- ----------- --------------------------
- * 10/15/10 6644 bphillip Initial Creation
- *
- *
- *
- * @author bphillip
- * @version 1
- * @see com.raytheon.edex.plugin.grib.GribLargeFileChecker
- * @see com.raytheon.edex.plugin.grib.GribLockRelease
- */
-@XmlRootElement(name = "largeGribPatterns")
-@XmlAccessorType(XmlAccessType.NONE)
-public class LargeGribPatterns implements ISerializableObject {
- /**
- * List of patterns
- */
- @XmlElements({ @XmlElement(name = "regex", type = String.class) })
- private List patterns;
-
- /** List of compiled patterns */
- private List compiledPatterns;
-
- /**
- * Creates a new instance of the container.
- */
- public LargeGribPatterns() {
- this.patterns = new ArrayList();
- this.compiledPatterns = new ArrayList();
- }
-
- /**
- * Returns a list of the stored patterns as a series of strings.
- *
- * @return a list of regex pattern strings
- */
- public List getPatterns() {
- return patterns;
- }
-
- /**
- * Sets the list of regex strings for this container.
- *
- * @param patterns
- * an arraylist of regex strings
- */
- public void setPatterns(ArrayList patterns) {
- this.patterns = patterns;
- }
-
- /**
- * Inserts a single string into the list.
- *
- * @param pattern
- * The regex string to insert
- */
- public void setPattern(String pattern) {
- this.patterns.add(pattern);
- }
-
- /**
- * Will compile the strings into Pattern objects.
- *
- */
- public void compilePatterns() {
- for (String pattern : patterns) {
- compiledPatterns.add(Pattern.compile(pattern));
- }
- }
-
- /**
- * Takes a string and compares against the patterns in this container. The
- * first one that matches breaks the search and returns true.
- *
- * @param header
- * The string to search for
- * @return a boolean indicating success
- */
- public boolean isDesiredHeader(String header) {
- boolean isFound = false;
- for (Pattern headerPattern : compiledPatterns) {
- if (headerPattern.matcher(header).find()) {
- isFound = true;
- break;
- }
- }
- return isFound;
- }
-}
diff --git a/edexOsgi/com.raytheon.edex.plugin.grib/utility/edex_static/base/grib/largeGribPatterns.xml b/edexOsgi/com.raytheon.edex.plugin.grib/utility/edex_static/base/grib/largeGribPatterns.xml
deleted file mode 100644
index 4bf2c4baf5..0000000000
--- a/edexOsgi/com.raytheon.edex.plugin.grib/utility/edex_static/base/grib/largeGribPatterns.xml
+++ /dev/null
@@ -1,25 +0,0 @@
-
-
-
- ^al*
- ^tpcprblty*
- ^LGX[TP]*
-
\ No newline at end of file