From 772138b74821258e6e1665cf2b0c63f4e1d38d4d Mon Sep 17 00:00:00 2001 From: David Gillingham Date: Mon, 8 Apr 2013 10:16:45 -0500 Subject: [PATCH] Issue #1761: Refactor SendGridsToNDFD to use ExportGridsRequest, refactor ExportGridsRequest to use enum for mode, revert NATIONAL_CENTER changes to svcbu export_grids script. Export grids cron will now read svcbu.properties for a PRIMARY_SITES setting to determine sites to export for. Change-Id: Ie9454f1de0f00f9ee89723599accc90ac1536fe0 Former-commit-id: 181bb2914ea8c9e4f20e442e78826336d52cf400 --- .../sbu/jobs/SvcbuExportDigitalDataJob.java | 8 +- .../res/spring/gfe-request.xml | 7 - .../handler/SendGridsToNDFDHandler.java | 65 --- .../svcbu/ExportGridsRequestHandler.java | 9 +- .../plugin/gfe/svcbackup/ExportGridsTask.java | 70 +++- .../gfe/request/ExportGridsRequest.java | 26 +- .../gfe/request/SendGridsToNDFDRequest.java | 68 ---- .../configuration/svcbu.properties | 7 + .../ServiceBackup/scripts/export_grids | 370 +++++++++--------- .../src/sendgridstondfd/SendGridsToNDFD.py | 22 +- .../gfe/request/ExportGridsRequest.py | 79 ++++ .../gfe/request/SendGridsToNDFDRequest.py | 33 -- .../common/dataplugin/gfe/request/__init__.py | 4 +- 13 files changed, 383 insertions(+), 385 deletions(-) delete mode 100644 edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SendGridsToNDFDHandler.java delete mode 100644 edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.java create mode 100644 pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.py delete mode 100644 pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.py diff --git a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/sbu/jobs/SvcbuExportDigitalDataJob.java b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/sbu/jobs/SvcbuExportDigitalDataJob.java index 613c5d35c2..ca4fa2a32d 100644 --- a/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/sbu/jobs/SvcbuExportDigitalDataJob.java +++ b/cave/com.raytheon.viz.gfe/src/com/raytheon/viz/gfe/dialogs/sbu/jobs/SvcbuExportDigitalDataJob.java @@ -20,6 +20,7 @@ package com.raytheon.viz.gfe.dialogs.sbu.jobs; import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest; +import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest.ExportGridsMode; import com.raytheon.uf.common.status.UFStatus.Priority; import com.raytheon.viz.gfe.GFEServerException; @@ -32,7 +33,9 @@ import com.raytheon.viz.gfe.GFEServerException; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Aug 5, 2011 bphillip Initial creation + * Aug 05, 2011 bphillip Initial creation + * Apr 30, 2013 #1761 dgilling Support changes made to + * ExportGridsRequest. * * * @@ -54,7 +57,8 @@ public class SvcbuExportDigitalDataJob extends ServiceBackupJob { @Override public void run() { - ExportGridsRequest request = new ExportGridsRequest(site, "-m"); + ExportGridsRequest request = new ExportGridsRequest(site, + ExportGridsMode.MANUAL); try { makeRequest(request); } catch (GFEServerException e) { diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml index 8ef903c923..f16fccc84c 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/res/spring/gfe-request.xml @@ -260,13 +260,6 @@ value="com.raytheon.uf.common.dataplugin.gfe.request.GetSiteTimeZoneInfoRequest" /> - - - - - diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SendGridsToNDFDHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SendGridsToNDFDHandler.java deleted file mode 100644 index 059f177421..0000000000 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/SendGridsToNDFDHandler.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.edex.plugin.gfe.server.handler; - -import com.raytheon.edex.plugin.gfe.svcbackup.SvcBackupUtil; -import com.raytheon.uf.common.dataplugin.gfe.request.SendGridsToNDFDRequest; -import com.raytheon.uf.common.dataplugin.gfe.server.message.ServerResponse; -import com.raytheon.uf.common.serialization.comm.IRequestHandler; - -/** - * TODO Add Description - * - * 
- * 
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Mar 17, 2010            avarani     Initial creation
- * 
- *
- * - * @author avarani - * @version 1.0 - */ - -public class SendGridsToNDFDHandler implements - IRequestHandler { - - /* - * (non-Javadoc) - * - * @see - * com.raytheon.uf.common.serialization.comm.IRequestHandler#handleRequest - * (com.raytheon.uf.common.serialization.comm.IServerRequest) - */ - @Override - public ServerResponse handleRequest(SendGridsToNDFDRequest request) - throws Exception { - ServerResponse response = new ServerResponse(); - String site = request.getSite(); - try { - SvcBackupUtil.execute("export_grids", "-m", site.toLowerCase()); - } catch (Exception e) { - response.addMessage(e.getMessage()); - } - return response; - } -} diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/svcbu/ExportGridsRequestHandler.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/svcbu/ExportGridsRequestHandler.java index 8184afe998..7a81d4207a 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/svcbu/ExportGridsRequestHandler.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/server/handler/svcbu/ExportGridsRequestHandler.java @@ -39,7 +39,9 @@ import com.raytheon.uf.common.serialization.comm.IRequestHandler; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Aug 4, 2011 bphillip Initial creation + * Aug 04, 2011 bphillip Initial creation + * Apr 30, 2013 #1761 dgilling Support changes made to + * ExportGridsRequest. * * * @@ -56,8 +58,9 @@ public class ExportGridsRequestHandler implements public Object handleRequest(ExportGridsRequest request) throws Exception { ServerResponse sr = new ServerResponse(); - SvcBackupUtil.execute("export_grids", request.getMode(), request - .getSite().toLowerCase()); + SvcBackupUtil.execute("export_grids", + request.getMode().getCmdLineArg(), request.getSite() + .toLowerCase()); ServiceBackupNotificationManager .sendMessageNotification("Digital data successfully exported."); diff --git a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/svcbackup/ExportGridsTask.java b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/svcbackup/ExportGridsTask.java index 7a6d94fcc6..8d01db8e0d 100644 --- a/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/svcbackup/ExportGridsTask.java +++ b/edexOsgi/com.raytheon.edex.plugin.gfe/src/com/raytheon/edex/plugin/gfe/svcbackup/ExportGridsTask.java @@ -19,12 +19,21 @@ **/ package com.raytheon.edex.plugin.gfe.svcbackup; +import java.util.ArrayList; +import java.util.Collection; import java.util.Date; +import java.util.Properties; +import java.util.Set; import java.util.TimerTask; +import com.google.common.collect.Sets; import com.raytheon.edex.plugin.gfe.server.handler.svcbu.ExportGridsRequestHandler; import com.raytheon.edex.site.SiteUtil; import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest; +import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest.ExportGridsMode; +import com.raytheon.uf.common.status.IUFStatusHandler; +import com.raytheon.uf.common.status.UFStatus; +import com.raytheon.uf.edex.site.SiteAwareRegistry; /** * TODO Add Description @@ -35,7 +44,9 @@ import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Aug 3, 2011 bphillip Initial creation + * Aug 03, 2011 bphillip Initial creation + * Apr 30, 2013 #1761 dgilling Read list of sites to export grids + * for from svcbu.properties. * * * @@ -45,6 +56,9 @@ import com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest; public class ExportGridsTask extends TimerTask { + private static final transient IUFStatusHandler statusHandler = UFStatus + .getHandler(ExportGridsTask.class); + public ExportGridsTask(Date executionTime) { ServiceBackupNotificationManager .sendMessageNotification("Export grids cron scheduled for execution at: " @@ -58,17 +72,51 @@ public class ExportGridsTask extends TimerTask { */ @Override public void run() { - ServiceBackupNotificationManager - .sendMessageNotification("Export Grids to central server cron started."); - try { - new ExportGridsRequestHandler() - .handleRequest(new ExportGridsRequest(SiteUtil.getSite(), - "-c")); - } catch (Exception e) { + final ExportGridsRequestHandler reqHandler = new ExportGridsRequestHandler(); + + for (String site : getSites()) { ServiceBackupNotificationManager - .sendErrorMessageNotification( - "Export Grids to central server cron failed to execute.", - e); + .sendMessageNotification("Export Grids to central server cron started for site " + + site + "."); + try { + reqHandler.handleRequest(new ExportGridsRequest(site, + ExportGridsMode.CRON)); + } catch (Exception e) { + ServiceBackupNotificationManager.sendErrorMessageNotification( + "Export Grids to central server cron failed to execute for site " + + site + ".", e); + } } } + + private Collection getSites() { + Properties svcbuProps = SvcBackupUtil.getSvcBackupProperties(); + String siteList = SiteUtil.getSite(); + if (svcbuProps != null) { + String propVal = svcbuProps.getProperty("PRIMARY_SITES", "").trim(); + if (!propVal.isEmpty()) { + siteList = propVal; + } + } + + String[] sites = siteList.split(","); + Collection retVal = new ArrayList(sites.length); + Set validSites = Sets.newHashSet(SiteAwareRegistry + .getInstance().getActiveSites()); + for (String site : sites) { + final String siteId = site.trim().toUpperCase(); + if (!siteId.isEmpty()) { + if (validSites.contains(siteId)) { + retVal.add(siteId); + } else { + statusHandler + .warn("Site " + + siteId + + " is not a currently activated site. Grids will not be exported for this site. Check the PRIMARY_SITES setting in svcbu.properties."); + } + } + } + + return retVal; + } } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.java index 6665f3205f..5c65ce345a 100644 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.java +++ b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.java @@ -31,7 +31,8 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; * * Date Ticket# Engineer Description * ------------ ---------- ----------- -------------------------- - * Aug 4, 2011 bphillip Initial creation + * Aug 04, 2011 bphillip Initial creation + * Apr 29, 2013 #1761 dgilling Make mode field an Enum. * * * @@ -42,17 +43,31 @@ import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; @DynamicSerialize public class ExportGridsRequest extends AbstractGfeRequest { + public enum ExportGridsMode { + CRON("-c"), MANUAL("-m"), GRIB2("-g"); + + private final String cmdLineArg; + + private ExportGridsMode(String cmdLineArg) { + this.cmdLineArg = cmdLineArg; + } + + public String getCmdLineArg() { + return cmdLineArg; + } + } + @DynamicSerializeElement private String site; @DynamicSerializeElement - private String mode; + private ExportGridsMode mode; public ExportGridsRequest() { } - public ExportGridsRequest(String site, String mode) { + public ExportGridsRequest(String site, ExportGridsMode mode) { this.site = site; this.mode = mode; } @@ -75,7 +90,7 @@ public class ExportGridsRequest extends AbstractGfeRequest { /** * @return the mode */ - public String getMode() { + public ExportGridsMode getMode() { return mode; } @@ -83,8 +98,7 @@ public class ExportGridsRequest extends AbstractGfeRequest { * @param mode * the mode to set */ - public void setMode(String mode) { + public void setMode(ExportGridsMode mode) { this.mode = mode; } - } diff --git a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.java b/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.java deleted file mode 100644 index 9acb3b55c0..0000000000 --- a/edexOsgi/com.raytheon.uf.common.dataplugin.gfe/src/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * This software was developed and / or modified by Raytheon Company, - * pursuant to Contract DG133W-05-CQ-1067 with the US Government. - * - * U.S. EXPORT CONTROLLED TECHNICAL DATA - * This software product contains export-restricted data whose - * export/transfer/disclosure is restricted by U.S. law. Dissemination - * to non-U.S. persons whether in the United States or abroad requires - * an export license or other authorization. - * - * Contractor Name: Raytheon Company - * Contractor Address: 6825 Pine Street, Suite 340 - * Mail Stop B8 - * Omaha, NE 68106 - * 402.291.0100 - * - * See the AWIPS II Master Rights File ("Master Rights File.pdf") for - * further licensing information. - **/ -package com.raytheon.uf.common.dataplugin.gfe.request; - -import com.raytheon.uf.common.serialization.annotations.DynamicSerialize; -import com.raytheon.uf.common.serialization.annotations.DynamicSerializeElement; -import com.raytheon.uf.common.serialization.comm.IServerRequest; - -/** - * TODO Add Description - * - * 
- * 
- * SOFTWARE HISTORY
- * Date         Ticket#    Engineer    Description
- * ------------ ---------- ----------- --------------------------
- * Mar 17, 2010            avarani     Initial creation
- * 
- *
- * - * @author avarani - * @version 1.0 - */ - -@DynamicSerialize -public class SendGridsToNDFDRequest implements IServerRequest { - /** - * Send the grids from this site. - */ - @DynamicSerializeElement - private String site; - - /** - * Getter for the site id. - * - * @return the site ID. - */ - public String getSite() { - return site; - } - - /** - * Setter for the site id. - * - * @param site - * send the grids from this site. - */ - public void setSite(String site) { - this.site = site; - } -} diff --git a/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/configuration/svcbu.properties b/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/configuration/svcbu.properties index 2405687e6d..fc3f23ca32 100644 --- a/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/configuration/svcbu.properties +++ b/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/configuration/svcbu.properties @@ -103,6 +103,13 @@ # SVCBU_USER_ID The user id of the designated user to run GFE when # in service backup # +# PRIMARY_SITES (Optional) For dual-domain sites, a comma-separated +# list of sites for the export grids cron to run for +# instead of the site defined as AW_SITE_IDENTIFIER. If +# this setting is empty or not defined, cron will only +# export grids for site set as AW_SITE_IDENTIFIER. +# +# # Directories used by Service Backup GFESUITE_HOME=${AWIPS_HOME}/GFESuite GFESUITE_BIN=${GFESUITE_HOME}/bin diff --git a/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/export_grids b/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/export_grids index d6efe8102a..91d7f844bc 100644 --- a/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/export_grids +++ b/edexOsgi/com.raytheon.uf.tools.gfesuite.servicebackup/svcBackup/ServiceBackup/scripts/export_grids @@ -1,27 +1,39 @@ #!/bin/bash +############################################################################## +# Script to export gridded data to the central server +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 04/29/13 #1761 dgilling Remove use of NATIONAL_CENTER, +# script caller will determine +# which sites to export. +############################################################################## + if [ ${#AWIPS_HOME} = 0 ] then path_to_script=`readlink -f $0` export AWIPS_HOME=$(dirname $(dirname $(dirname $(dirname $path_to_script)))) fi -# $1 = Run Mode -# $2 = Site name - +# ARGUMENTS +# $1 = -c run by cron +# -g run by grib2.bat (used by SendGridToNDFD) +# -m run by manual push +# also determined by EXPORT_GRID +# $2 = wfo ID. +# +# if [ $# -ne 2 ] then - log_msg "Incorrect number of arguments\nCorrect usage: export_grids [-c|-m] wfo" + log_msg "Incorrect number of arguments\nCorrect usage: export_grids [-c|-m|-g] wfo" exit 1 fi . ${AWIPS_HOME}/GFESuite/ServiceBackup/configuration/svcbu.env -#while [ "$(ls -A ${LOCK_DIR})" ] -#do -# sleep 5 -#done - # Create the log file logdir=${IFPS_LOG}/`date +%Y%m%d` logfil=svcbu_export_grids_`date +%H%M` @@ -30,197 +42,183 @@ logfile=${logdir}/${logfil} touch ${logdir}/${logfil} exec 1>${logdir}/${logfil} 2>&1 -# DR14464 - netcdf file creation is now a function to allow it to be called more than once for -# national centers -perform_export() { - SITE=`echo ${1} | tr '[A-Z]' '[a-z]'` - CAPS_SITE=`echo ${1} | tr [a-z] [A-Z]` - export SITE +SITE=`echo ${2} | tr '[A-Z]' '[a-z]'` +CAPS_SITE=`echo ${2} | tr [a-z] [A-Z]` +export SITE - manuallockFile=${LOCK_DIR}/${SITE}exportGrids - cronLockFile=${LOCK_DIR}/${SITE}exportGridscron - lockFile=${manuallockFile} - LOGGER=log_msg +manuallockFile=${LOCK_DIR}/${SITE}exportGrids +cronLockFile=${LOCK_DIR}/${SITE}exportGridscron +lockFile=${manuallockFile} +LOGGER=log_msg - if [ $1 = "-c" ] - then - LOGGER=echo_msg - lockFile=${cronLockFile} - fi +if [ $1 = "-c" ] +then + LOGGER=echo_msg + lockFile=${cronLockFile} +fi - # Check the status of the lock file to see if we are OK to proceed +#determine if export grid is allowed +# +if [ $EXPORT_GRID -eq 0 ] +then + $LOGGER "You turned off the export grid option. Check EXPORT_GRID." + $LOGGER 100 + exit 1 +elif [ "$1" = "-c" ] && [ $EXPORT_GRID -ne 1 ] +then + $LOGGER "Cannot execute grid export since you have disabled the cron job. Check EXPORT_GRID." + $LOGGER 100 + exit 1 +elif [ -z "$EXPORT_GRID" ] +then + $LOGGER "EXPORT_GRID not defined. Export grids failed" + $LOGGER 100 + exit 1 +fi - if [ -f ${manuallockFile} ] - then - $LOGGER Cannot export grids for ${CAPS_SITE}. Export grids process already in progress! Cron will not run this hour. - exit 1 - fi +if [ -f ${manuallockFile} ] +then + $LOGGER Cannot export grids for ${CAPS_SITE}. Export grids process already in progress! Cron will not run this hour. + exit 1 +fi + +touch ${lockFile} + +$LOGGER "Starting exporting grids..." +$LOGGER 0 + +# +# If this is a non-wfo/non-rfc site, they may not have an edit area for clipping - handle it +# +if [ "$SVCBU_GRIDAREA" == "" ] ; then + SVCBU_MASK="" +else + SVCBU_MASK="-m $SVCBU_GRIDAREA" +fi + +# Need to create the start and end time arguments for ifpnetCDF +# The start time needs to be 6 hours prior to current time, and end time needs to be 6 hours after the 9th day time. +starttime=`date -d "24 hours ago" +%Y%m%d`_`date -d "24 hours ago" +%H00` +endtime=`date -d "+8 days 6 hours" +%Y%m%d`_`date -d "+8 days 6 hours" +%H00` +# +$LOGGER 25 +$LOGGER "Running ifpnetCDF..." + +# Name of the database to use in ifpServer to grab grids from. +# NOTE: SVCBU_DB must be defined in ifps-ccc.env file. Defaults to Official. +if [ -n "${SVCBU_DB}" ] +then + DB_NAME=${CAPS_SITE}_GRID__${SVCBU_DB}_00000000_0000 +else + DB_NAME=${CAPS_SITE}_GRID__Official_00000000_0000 +fi +$LOGGER ${DB_NAME} - touch ${lockFile} +NETCDF_PATH=${GFESUITE_HOME}/exportgrids +export NETCDF_TMP_PATH=${NETCDF_PATH}/tmp +if [ ! -d ${NETCDF_TMP_PATH} ]; then + mkdir ${NETCDF_TMP_PATH} + chmod 777 ${NETCDF_TMP_PATH} +fi - $LOGGER "Starting exporting grids..." - $LOGGER 0 +# Implement wx element trimming. +# For this, we'll read from a flat file in ${IFPS_DATA} that will contain list +# of element a site wants to include. We'll also check if the site has +# $SVCBU_TRIM_ELEMS variable set to 1. We will only do wx element trimming if +# this variable is set to 1. Otherwise, we'll continue to send all grids. +NETCDF_SUCCESS=1 - # - # If this is a non-wfo/non-rfc site, they may not have an edit area for clipping - handle it - # - if [ "$SVCBU_GRIDAREA" == "" ] ; then - SVCBU_MASK="" - else - SVCBU_MASK="-m $SVCBU_GRIDAREA" - fi - - # Need to create the start and end time arguments for ifpnetCDF - # The start time needs to be 6 hours prior to current time, and end time needs to be 6 hours after the 9th day time. - starttime=`date -d "24 hours ago" +%Y%m%d`_`date -d "24 hours ago" +%H00` - endtime=`date -d "+8 days 6 hours" +%Y%m%d`_`date -d "+8 days 6 hours" +%H00` - # - $LOGGER 25 - $LOGGER "Running ifpnetCDF..." - # Name of the database to use in ifpServer to grab grids from. - # NOTE: SVCBU_DB must be defined in ifps-ccc.env file. Defaults to Official. - if [ -n "${SVCBU_DB}" ] - then - DB_NAME=${CAPS_SITE}_GRID__${SVCBU_DB}_00000000_0000 - else - DB_NAME=${CAPS_SITE}_GRID__Official_00000000_0000 - fi - - $LOGGER ${DB_NAME} - - - NETCDF_PATH=${GFESUITE_HOME}/exportgrids - export NETCDF_TMP_PATH=${NETCDF_PATH}/tmp - if [ ! -d ${NETCDF_TMP_PATH} ]; then - mkdir ${NETCDF_TMP_PATH} - chmod 777 ${NETCDF_TMP_PATH} - fi - - # Implement wx element trimming. - # For this, we'll read from a flat file in ${IFPS_DATA} that will contain list of element a site wants to include. - # We'll also check if the site has $SVCBU_TRIM_ELEMS variable set to 1. We will only do wx element trimming if this variable is set to 1. - # Otherwise, we'll continue to send all grids. - NETCDF_SUCCESS=1 - - if [ "${SVCBU_TRIM_ELEMS}" != "" -a "${SVCBU_TRIM_ELEMS}" = "1" ] - then - # Check if we have the file that has list of elements to trim for. - if [ -f ${IFPS_DATA}/svcbu_export_elements.${SITE} ]; then - # Building the -p arguments to ifpnetCDF. - wxflags="" - for elem in `cat ${IFPS_DATA}/svcbu_export_elements.${SITE}` - do - wxflags="$wxflags -p ${elem}" - done - #Now run ifpnetCDF with $wxflags as well as time flags. - ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h ${SVCBU_HOST} -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} ${wxflags} - #check if the netcdf file is there;if not,exit - if [ $? -ne 0 ] - then - NETCDF_SUCCESS=0 - if [ $NATIONAL_CENTER != 1 ] && [ "$1" != "-c" ] - then - netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` - if [ "$netcdf_error" != "" ]; then - NETCDF_SUCCESS=1 - else - $LOGGER "ifpnetCDF failed. Export exits." - rm -f ${lockFile} - $LOGGER 100 - exit 1 - fi - fi - fi - else - $LOGGER "export_grids was not able to find ${IFPS_DATA}/svcbu_export_elements.${SITE} file." - $LOGGER "Without this file, ifpnetCDF will not know how to trim for needed weather elements." - $LOGGER "ifpnetCDF will run without doing any trimming." - #Starting ifpnetCDF to pack the grids (netcdf file is stored in the IFPS_LOG dir) - ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h $SVCBU_HOST -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} - #check if the netcdf file is there;if not,exit - if [ $? -ne 0 ] - then - NETCDF_SUCCESS=0 - if [ $NATIONAL_CENTER != 1 ] && [ "$1" != "-c" ] - then - netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` - if [ "$netcdf_error" != "" ]; then - NETCDF_SUCCESS=1 - else - $LOGGER "ifpnetCDF failed. Export exits." - rm -f ${lockFile} - $LOGGER 100 - exit 1 - fi - fi - fi - fi - else - # NOTE: This is the case when no wx elems trimming will done. ifpnetCDF file will contain everything. - #Starting ifpnetCDF to pack the grids (netcdf file is stored in the IFPS_LOG dir) - ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h $SVCBU_HOST -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} - #check if the netcdf file is there;if not,exit before we call msg_send +if [ "${SVCBU_TRIM_ELEMS}" != "" -a "${SVCBU_TRIM_ELEMS}" = "1" ] +then + # Check if we have the file that has list of elements to trim for. + if [ -f ${IFPS_DATA}/svcbu_export_elements.${SITE} ]; then + # Building the -p arguments to ifpnetCDF. + wxflags="" + for elem in `cat ${IFPS_DATA}/svcbu_export_elements.${SITE}` + do + wxflags="$wxflags -p ${elem}" + done + #Now run ifpnetCDF with $wxflags as well as time flags. + ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h ${SVCBU_HOST} -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} ${wxflags} + #check if the netcdf file is there;if not,exit if [ $? -ne 0 ] then - NETCDF_SUCCESS=0 - if [ $NATIONAL_CENTER != 1 ] && [ "$1" != "-c" ] - then - netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` - if [ "$netcdf_error" != "" ]; then - NETCDF_SUCCESS=1 - else - $LOGGER "ifpnetCDF failed." - rm -f ${lockFile} - $LOGGER 100 - exit 1 - fi - fi + NETCDF_SUCCESS=0 + if [ "$1" != "-c" ] + then + netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` + if [ "$netcdf_error" != "" ]; then + NETCDF_SUCCESS=1 + else + $LOGGER "ifpnetCDF failed. Export exits." + rm -f ${lockFile} + $LOGGER 100 + exit 1 + fi + fi fi - fi + else + $LOGGER "export_grids was not able to find ${IFPS_DATA}/svcbu_export_elements.${SITE} file." + $LOGGER "Without this file, ifpnetCDF will not know how to trim for needed weather elements." + $LOGGER "ifpnetCDF will run without doing any trimming." + #Starting ifpnetCDF to pack the grids (netcdf file is stored in the IFPS_LOG dir) + ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h $SVCBU_HOST -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} + #check if the netcdf file is there;if not,exit + if [ $? -ne 0 ] + then + NETCDF_SUCCESS=0 + if [ "$1" != "-c" ] + then + netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` + if [ "$netcdf_error" != "" ]; then + NETCDF_SUCCESS=1 + else + $LOGGER "ifpnetCDF failed. Export exits." + rm -f ${lockFile} + $LOGGER 100 + exit 1 + fi + fi + fi + fi +else + # NOTE: This is the case when no wx elems trimming will done. ifpnetCDF file will contain everything. + #Starting ifpnetCDF to pack the grids (netcdf file is stored in the IFPS_LOG dir) + ${GFESUITE_BIN}/ifpnetCDF -o ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf -h $SVCBU_HOST -r ${CDSPORT} -d ${DB_NAME} -k -t ${SVCBU_MASK} + #check if the netcdf file is there;if not,exit before we call msg_send + if [ $? -ne 0 ] + then + NETCDF_SUCCESS=0 + if [ "$1" != "-c" ] + then + netcdf_error=`grep "Some of the requested time ranges are not in the inventory" $logfile` + if [ "$netcdf_error" != "" ]; then + NETCDF_SUCCESS=1 + else + $LOGGER "ifpnetCDF failed." + rm -f ${lockFile} + $LOGGER 100 + exit 1 + fi + fi + fi +fi - if [ $NETCDF_SUCCESS = 1 ] - then - $LOGGER "Copying ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf to ${NETCDF_PATH}/${SITE}Grd.netcdf" - cp ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf ${NETCDF_PATH}/${SITE}Grd.netcdf - fi - - if [ -f ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf ] - then - $LOGGER "Removing ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf" - rm -f ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf - fi - - rm -f ${lockFile} - $LOGGER 100 -} -# end DR14464 change - -if [ $EXPORT_GRID = 0 ] +if [ $NETCDF_SUCCESS = 1 ] then - log_msg "You turned off the export grid option. Check EXPORT_GRID" - log_msg 100 - exit 1 -elif [ $EXPORT_GRID = 2 ] && [ "$1" = "-c" ] + $LOGGER "Copying ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf to ${NETCDF_PATH}/${SITE}Grd.netcdf" + cp ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf ${NETCDF_PATH}/${SITE}Grd.netcdf +fi + +if [ -f ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf ] then - log_msg "export_grids cron disabled" - exit 1 -elif [ $NATIONAL_CENTER = 1 ] && [ "$1" = "-c" ] # DR14464 changes - loop through active sites -then - log_msg "You are configured as $AW_SITE_IDENTIFIER. Will use activeSites.txt for site id info." - site_list=( `cat $LOCALIZATION_PATH/edex_static/site/$AW_SITE_IDENTIFIER/config/activeSites.txt` ) - for site in "${site_list[@]}" - do - if [ $site != $AW_SITE_IDENTIFIER ] - then - log_msg "Processing $site" - perform_export ${site} - fi - done -else # this processes non-national centers the old way using AW_SITE_IDENTIFIER if run by cron - log_msg "Processing $2" - perform_export ${2} -fi # end DR14464 changes + $LOGGER "Removing ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf" + rm -f ${NETCDF_TMP_PATH}/${SITE}Grd.netcdf +fi + +rm -f ${lockFile} +$LOGGER 100 # Completed successfully $LOGGER "All done. Export Gridded data has completed" diff --git a/edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgridstondfd/SendGridsToNDFD.py b/edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgridstondfd/SendGridsToNDFD.py index ed579a9f25..88ad46022b 100644 --- a/edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgridstondfd/SendGridsToNDFD.py +++ b/edexOsgi/com.raytheon.uf.tools.gfesuite/cli/src/sendgridstondfd/SendGridsToNDFD.py @@ -18,10 +18,27 @@ # further licensing information. ## +# +# Provides a command-line utility to a site's grids to a folder and later +# sent to NDFD. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# ??/??/?? ???????? Initial Creation. +# 04/05/13 1761 dgilling Rewrite based on +# ExportGridsRequest. +# +# +# + + import argparse import sys -from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import SendGridsToNDFDRequest +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import ExportGridsRequest from ufpy import ThriftClient from ufpy import UsageArgumentParser @@ -31,8 +48,9 @@ class SendGridsToNDFD: def send(self, site, host, port): thriftClient = ThriftClient.ThriftClient(host, port) - ndfdRequest = SendGridsToNDFDRequest() + ndfdRequest = ExportGridsRequest() ndfdRequest.setSite(site) + ndfdRequest.setMode("GRIB2") serverResponse = thriftClient.sendRequest(ndfdRequest) if (not serverResponse.isOkay()): diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.py new file mode 100644 index 0000000000..defa9e5c32 --- /dev/null +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/ExportGridsRequest.py @@ -0,0 +1,79 @@ +## +# This software was developed and / or modified by Raytheon Company, +# pursuant to Contract DG133W-05-CQ-1067 with the US Government. +# +# U.S. EXPORT CONTROLLED TECHNICAL DATA +# This software product contains export-restricted data whose +# export/transfer/disclosure is restricted by U.S. law. Dissemination +# to non-U.S. persons whether in the United States or abroad requires +# an export license or other authorization. +# +# Contractor Name: Raytheon Company +# Contractor Address: 6825 Pine Street, Suite 340 +# Mail Stop B8 +# Omaha, NE 68106 +# 402.291.0100 +# +# See the AWIPS II Master Rights File ("Master Rights File.pdf") for +# further licensing information. +## + +# +# A pure python implementation of com.raytheon.uf.common.dataplugin.gfe.request.ExportGridsRequest +# for use by the python implementation of DynamicSerialize. +# +# File auto-generated against equivalent DynamicSerialize Java class, but additional +# useful methods have been added. +# +# +# SOFTWARE HISTORY +# +# Date Ticket# Engineer Description +# ------------ ---------- ----------- -------------------------- +# 04/05/13 dgilling Initial Creation. +# +# +# + +from dynamicserialize.dstypes.com.raytheon.uf.common.dataplugin.gfe.request import AbstractGfeRequest + + +class ExportGridsRequest(AbstractGfeRequest): + + def __init__(self): + super(ExportGridsRequest, self).__init__() + self.site = None + self.mode = None + + def getSite(self): + return self.site + + def setSite(self, site): + self.site = site + + def getMode(self): + return self.mode + + def setMode(self, mode): + validValues = ['CRON', 'MANUAL', 'GRIB2'] + inputVal = str(mode).upper() + if inputVal in validValues: + self.mode = mode + else: + raise ValueError(inputVal + " not a valid ExportGridsMode value. Must be one of " + str(validValues)) + + def __str__(self): + retVal = "ExportGridsRequest[" + retVal += "wokstationID: " + str(self.workstationID) + ", " + retVal += "siteID: " + str(self.siteID) + ", " + retVal += "site: " + str(self.site) + ", " + retVal += "mode: " + str(self.mode) + "]" + return retVal + + def __repr__(self): + retVal = "ExportGridsRequest(" + retVal += "wokstationID=" + repr(self.workstationID) + ", " + retVal += "siteID=" + repr(self.siteID) + ", " + retVal += "site=" + repr(self.site) + ", " + retVal += "mode=" + repr(self.mode) + ")" + return retVal diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.py deleted file mode 100644 index b27b44048b..0000000000 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/SendGridsToNDFDRequest.py +++ /dev/null @@ -1,33 +0,0 @@ -## -# This software was developed and / or modified by Raytheon Company, -# pursuant to Contract DG133W-05-CQ-1067 with the US Government. -# -# U.S. EXPORT CONTROLLED TECHNICAL DATA -# This software product contains export-restricted data whose -# export/transfer/disclosure is restricted by U.S. law. Dissemination -# to non-U.S. persons whether in the United States or abroad requires -# an export license or other authorization. -# -# Contractor Name: Raytheon Company -# Contractor Address: 6825 Pine Street, Suite 340 -# Mail Stop B8 -# Omaha, NE 68106 -# 402.291.0100 -# -# See the AWIPS II Master Rights File ("Master Rights File.pdf") for -# further licensing information. -## - -# File auto-generated against equivalent DynamicSerialize Java class - -class SendGridsToNDFDRequest(object): - - def __init__(self): - self.site = None - - def getSite(self): - return self.site - - def setSite(self, site): - self.site = site - diff --git a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py index 6efb901e5b..864f663707 100644 --- a/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py +++ b/pythonPackages/dynamicserialize/dstypes/com/raytheon/uf/common/dataplugin/gfe/request/__init__.py @@ -27,6 +27,7 @@ __all__ = [ 'ConfigureTextProductsRequest', 'ExecuteIfpNetCDFGridRequest', 'ExecuteIscMosaicRequest', + 'ExportGridsRequest', 'GetASCIIGridsRequest', 'GetGridDataRequest', 'GetGridInventoryRequest', @@ -44,7 +45,6 @@ __all__ = [ 'ProcessReceivedDigitalDataRequest', 'PurgeGfeGridsRequest', 'SaveASCIIGridsRequest', - 'SendGridsToNDFDRequest', 'ServiceBackupStatusUpdateRequest', 'SmartInitRequest' ] @@ -55,6 +55,7 @@ from CommitGridsRequest import CommitGridsRequest from ConfigureTextProductsRequest import ConfigureTextProductsRequest from ExecuteIfpNetCDFGridRequest import ExecuteIfpNetCDFGridRequest from ExecuteIscMosaicRequest import ExecuteIscMosaicRequest +from ExportGridsRequest import ExportGridsRequest from GetASCIIGridsRequest import GetASCIIGridsRequest from GetGridDataRequest import GetGridDataRequest from GetGridInventoryRequest import GetGridInventoryRequest @@ -72,7 +73,6 @@ from ProcessReceivedConfRequest import ProcessReceivedConfRequest from ProcessReceivedDigitalDataRequest import ProcessReceivedDigitalDataRequest from PurgeGfeGridsRequest import PurgeGfeGridsRequest from SaveASCIIGridsRequest import SaveASCIIGridsRequest -from SendGridsToNDFDRequest import SendGridsToNDFDRequest from ServiceBackupStatusUpdateRequest import ServiceBackupStatusUpdateRequest from SmartInitRequest import SmartInitRequest