Issue #1940 - MPE performance and set_hydro_env change

Former-commit-id: b8f8f0e7b1 [formerly 4d9db6d27437c6bc0a1886d41b2451c1cb714ad0]
Former-commit-id: 48ecfd2124
This commit is contained in:
Mike Duff 2013-04-30 09:28:49 -05:00
parent 4f5785e0db
commit 0c55fd80cf
9 changed files with 5 additions and 527 deletions

View file

@ -67,9 +67,9 @@ integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 2) ]]
then
sleep 2
sleep 1
CNTR=CNTR+1
else
break

View file

@ -67,9 +67,9 @@ integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 2) ]]
then
sleep 2
sleep 1
CNTR=CNTR+1
else
break

View file

@ -84,6 +84,7 @@ do
PGSQL_DRIVER_JAR=$i
done
export DB_DRIVER_PATH=${DB_DRIVER_DIR}/${PGSQL_DRIVER_JAR}
cd -
#===========================================================================
# This is stuff extracted from the original HYDRO_ENV

View file

@ -1,77 +0,0 @@
#!/bin/ksh
# file name: run_build_hourly lookback_window end_time
# lookback_window & end_time is optional, format is YYYY-MM-DD HH:MM:SS
# if end_time is not specified then the current time will be assumed
# Created: 23 June 2005 Author : Varalakshmi Rajaram
# This allows you to run this script from outside of ./bin
RUN_FROM_DIR=`dirname $0`
# set up SOME environment variables for WHFS applications
. $RUN_FROM_DIR/../../../set_hydro_env
export BUILD_HOURLY_ENABLE=$(get_apps_defaults build_hourly_enable)
export GAGE_PP_DATA=$(get_apps_defaults gage_pp_data)
export GAGE_PP_LOG=$(get_apps_defaults gage_pp_log)
export DB_NAME=$(get_apps_defaults db_name)
TEMP_DATE=`date -u +%m%d`
BUILD_HOURLY_LOGFILE=$GAGE_PP_LOG/build_hourly_$TEMP_DATE.log
if [[ $BUILD_HOURLY_ENABLE = OFF ]]
then
echo "Build hourly is not enabled...Exiting" >>$BUILD_HOURLY_LOGFILE
exit 1
fi
#Usage
USAGE="run_build_hourly lookback_window_in_hrs <end_time as \"YYYY-MM-DD HH:MM:SS\"> or run_build_hourly"
if [[ ! ( ($# -eq 2) || ($# -eq 0) ) ]]
then
print -u2 "\nUSAGE: $USAGE\n"
exit 1
fi
if [[ $# -eq 0 ]]
then
LOOKBACK_WINDOW=3 #default num of hours for which the hourly total is calculated
else
LOOKBACK_WINDOW=$1 #number of hours for which the hourly total is calculated
fi
# Command line arguments to the program build_hourly
END_TIME="$2"
DATABASE_NAME=$DB_NAME
MIN_PERCENT_FILLED=0.75 # Create the load file for the hour only if
# the data obtained by summation of the sub
# hourly data covers the whole hour
TYPE_HANDLING=PTYPE # SAME / PTYPE
# SAME - Retain the org type source
# PTYPE - Replace the R in RG, RZ,
# RR etc to P
echo "******************************" >>$BUILD_HOURLY_LOGFILE
echo "Started Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
# build_hourly -d dbname -l lookback_window -p percent_filled
# -t type_source [-e end_time]
runso rary.ohd.pproc build_hourly_main -d $DATABASE_NAME -l $LOOKBACK_WINDOW \
-p $MIN_PERCENT_FILLED -t $TYPE_HANDLING -e "$END_TIME" >>$BUILD_HOURLY_LOGFILE 2>&1
echo " " >>$BUILD_HOURLY_LOGFILE
echo "Stopped Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
echo "******************************" >>$BUILD_HOURLY_LOGFILE
integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
then
sleep 2
CNTR=CNTR+1
else
break
fi
done

View file

@ -1,77 +0,0 @@
#!/bin/ksh
# file name: run_build_hourly lookback_window end_time
# lookback_window & end_time is optional, format is YYYY-MM-DD HH:MM:SS
# if end_time is not specified then the current time will be assumed
# Created: 23 June 2005 Author : Varalakshmi Rajaram
# This allows you to run this script from outside of ./bin
RUN_FROM_DIR=`dirname $0`
# set up SOME environment variables for WHFS applications
. $RUN_FROM_DIR/../../../set_hydro_env
export BUILD_HOURLY_ENABLE=$(get_apps_defaults build_hourly_enable)
export GAGE_PP_DATA=$(get_apps_defaults gage_pp_data)
export GAGE_PP_LOG=$(get_apps_defaults gage_pp_log)
export DB_NAME=$(get_apps_defaults db_name)
TEMP_DATE=`date -u +%m%d`
BUILD_HOURLY_LOGFILE=$GAGE_PP_LOG/build_hourly_$TEMP_DATE.log
if [[ $BUILD_HOURLY_ENABLE = OFF ]]
then
echo "Build hourly is not enabled...Exiting" >>$BUILD_HOURLY_LOGFILE
exit 1
fi
#Usage
USAGE="run_build_hourly lookback_window_in_hrs <end_time as \"YYYY-MM-DD HH:MM:SS\"> or run_build_hourly"
if [[ ! ( ($# -eq 2) || ($# -eq 0) ) ]]
then
print -u2 "\nUSAGE: $USAGE\n"
exit 1
fi
if [[ $# -eq 0 ]]
then
LOOKBACK_WINDOW=3 #default num of hours for which the hourly total is calculated
else
LOOKBACK_WINDOW=$1 #number of hours for which the hourly total is calculated
fi
# Command line arguments to the program build_hourly
END_TIME="$2"
DATABASE_NAME=$DB_NAME
MIN_PERCENT_FILLED=0.75 # Create the load file for the hour only if
# the data obtained by summation of the sub
# hourly data covers the whole hour
TYPE_HANDLING=PTYPE # SAME / PTYPE
# SAME - Retain the org type source
# PTYPE - Replace the R in RG, RZ,
# RR etc to P
echo "******************************" >>$BUILD_HOURLY_LOGFILE
echo "Started Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
# build_hourly -d dbname -l lookback_window -p percent_filled
# -t type_source [-e end_time]
runso rary.ohd.pproc build_hourly_main -d $DATABASE_NAME -l $LOOKBACK_WINDOW \
-p $MIN_PERCENT_FILLED -t $TYPE_HANDLING -e "$END_TIME" >>$BUILD_HOURLY_LOGFILE 2>&1
echo " " >>$BUILD_HOURLY_LOGFILE
echo "Stopped Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
echo "******************************" >>$BUILD_HOURLY_LOGFILE
integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
then
sleep 2
CNTR=CNTR+1
else
break
fi
done

View file

@ -1,103 +0,0 @@
#! /bin/ksh
# set_hydro_env
#
# This script sets up the environment so that ohdlib based programs can be
# executed from the command line.
#
#AWIPS_MODIFICATION_BLOCK_BEGIN
# locate EDEX_HOME
if [ "${EDEX_HOME}" = "" ]; then
default_edex_home=$(cd $(dirname $0)/../../../../edex;pwd)
if test -d $default_edex_home; then
export EDEX_HOME=$default_edex_home
else
echo "Unable to locate EDEX_HOME"
exit 1
fi
fi
if [ "${LOCALIZATION_ROOT}" = "" ]; then
export LOCALIZATION_ROOT=${EDEX_HOME}/data/utility/common_static
fi
if [ "${apps_dir}" = "" ]; then
export apps_dir=${EDEX_HOME}/data/hdf5/hydroapps
fi
# update the path to include the location of the runso rary.ohd.utility
export PATH=$PATH:${EDEX_HOME}/bin:${apps_dir}/bin
# contains the path to the ohd libraries and JNI-related library/libraries
export LD_LIBRARY_PATH=${EDEX_HOME}/lib/native/linux32:${apps_dir}/lib/native/linux32:$LD_LIBRARY_PATH
export RFCLX=${apps_dir}
#AWIPS_MODIFICATION_BLOCK_END
export POSTGRESQLDIR=/usr
export POSTGRESQLBINDIR=/awips2/psql/bin
export APPS_DEFAULTS_SITE=${LOCALIZATION_ROOT}/site/OAX/hydro/Apps_defaults
export APPS_DEFAULTS=${LOCALIZATION_ROOT}/base/hydro/Apps_defaults
# define a get_apps_defaults function for retrieving application token values
get_apps_defaults() {
runso rary.ohd.util gad $1
}
# OS_SUFFIX is the suffix used for filenames to identify the operating system
export OS_SUFFIX=".LX"
export geo_data=$RFCLX/geo_data
export PGOPTIONS='-c standard_conforming_strings=on'
export FXA_HOME=/awips/fxa
export FXA_BIN_DIR=$FXA_HOME/bin
# The following variables provide a JDBC URL or pieces thereof for
# SSHP and any other java programs that access the IHFS database
# The final form of the url is something like
# jdbc:postgresql://dx1:5432/hd_ob6abc?user=oper
export PGHOST=$(get_apps_defaults pghost)
export PGPORT=$(get_apps_defaults pgport)
export DB_NAME=$(get_apps_defaults db_name)
export PGUSER=$(get_apps_defaults pguser)
export DAMCAT_DB_NAME=$(get_apps_defaults damcat_db_name)
export JDBCURL=jdbc:postgresql://$PGHOST:$PGPORT/$DB_NAME?user=$PGUSER
export DAMCREST_JDBCURL=jdbc:postgresql://$PGHOST:$PGPORT/$DAMCAT_DB_NAME?user=$PGUSER
PGSQL_DRIVER_JAR=postgresql-8.3-603.jdbc3.jar
if [ "${PGSQL_DRIVER_DIR}" = "" ]; then
export DB_DRIVER_PATH=${EDEX_HOME}/lib/dependencies/org.postgres/${PGSQL_DRIVER_JAR}
else
export DB_DRIVER_PATH=${PGSQL_DRIVER_DIR}/${PGSQL_DRIVER_JAR}
fi
#===========================================================================
# This is stuff extracted from the original HYDRO_ENV
# and are used in run and start scripts and other files
#===========================================================================
# The following variable is needed for lots of scripts
export WHFS_BIN_DIR=$(get_apps_defaults whfs_bin_dir)
export WHFS_LOCAL_BIN_DIR=$(get_apps_defaults whfs_local_bin_dir)
#These variables needed to establish path to input and output files to the model in the DamCrest Application
export WHFS_LOCAL_DATA_DIR=$(get_apps_defaults whfs_local_data_dir)
export DAMCREST_DATA_DIR=$(get_apps_defaults damcrest_data_dir)
# this variable is needed to find gvim resource file .vimrc
export DAMCREST_RES_DIR=$(get_apps_defaults damcrest_res_dir)
# The following variable is needed for DamCat
export WFO_HELP_DIR=$(get_apps_defaults standard_data_dir)/help
# the following are needed for River/Precip Monitor
export RPF_LINE_WIDTH=$(get_apps_defaults rpf_linewidth)
export WHFS_UTIL_LOG_DIR=$(get_apps_defaults whfs_util_log_dir)
umask 000

View file

@ -1,77 +0,0 @@
#!/bin/ksh
# file name: run_build_hourly lookback_window end_time
# lookback_window & end_time is optional, format is YYYY-MM-DD HH:MM:SS
# if end_time is not specified then the current time will be assumed
# Created: 23 June 2005 Author : Varalakshmi Rajaram
# This allows you to run this script from outside of ./bin
RUN_FROM_DIR=`dirname $0`
# set up SOME environment variables for WHFS applications
. $RUN_FROM_DIR/../../../set_hydro_env
export BUILD_HOURLY_ENABLE=$(get_apps_defaults build_hourly_enable)
export GAGE_PP_DATA=$(get_apps_defaults gage_pp_data)
export GAGE_PP_LOG=$(get_apps_defaults gage_pp_log)
export DB_NAME=$(get_apps_defaults db_name)
TEMP_DATE=`date -u +%m%d`
BUILD_HOURLY_LOGFILE=$GAGE_PP_LOG/build_hourly_$TEMP_DATE.log
if [[ $BUILD_HOURLY_ENABLE = OFF ]]
then
echo "Build hourly is not enabled...Exiting" >>$BUILD_HOURLY_LOGFILE
exit 1
fi
#Usage
USAGE="run_build_hourly lookback_window_in_hrs <end_time as \"YYYY-MM-DD HH:MM:SS\"> or run_build_hourly"
if [[ ! ( ($# -eq 2) || ($# -eq 0) ) ]]
then
print -u2 "\nUSAGE: $USAGE\n"
exit 1
fi
if [[ $# -eq 0 ]]
then
LOOKBACK_WINDOW=3 #default num of hours for which the hourly total is calculated
else
LOOKBACK_WINDOW=$1 #number of hours for which the hourly total is calculated
fi
# Command line arguments to the program build_hourly
END_TIME="$2"
DATABASE_NAME=$DB_NAME
MIN_PERCENT_FILLED=0.75 # Create the load file for the hour only if
# the data obtained by summation of the sub
# hourly data covers the whole hour
TYPE_HANDLING=PTYPE # SAME / PTYPE
# SAME - Retain the org type source
# PTYPE - Replace the R in RG, RZ,
# RR etc to P
echo "******************************" >>$BUILD_HOURLY_LOGFILE
echo "Started Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
# build_hourly -d dbname -l lookback_window -p percent_filled
# -t type_source [-e end_time]
runso rary.ohd.pproc build_hourly_main -d $DATABASE_NAME -l $LOOKBACK_WINDOW \
-p $MIN_PERCENT_FILLED -t $TYPE_HANDLING -e "$END_TIME" >>$BUILD_HOURLY_LOGFILE 2>&1
echo " " >>$BUILD_HOURLY_LOGFILE
echo "Stopped Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
echo "******************************" >>$BUILD_HOURLY_LOGFILE
integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
then
sleep 2
CNTR=CNTR+1
else
break
fi
done

View file

@ -1,77 +0,0 @@
#!/bin/ksh
# file name: run_build_hourly lookback_window end_time
# lookback_window & end_time is optional, format is YYYY-MM-DD HH:MM:SS
# if end_time is not specified then the current time will be assumed
# Created: 23 June 2005 Author : Varalakshmi Rajaram
# This allows you to run this script from outside of ./bin
RUN_FROM_DIR=`dirname $0`
# set up SOME environment variables for WHFS applications
. $RUN_FROM_DIR/../../../set_hydro_env
export BUILD_HOURLY_ENABLE=$(get_apps_defaults build_hourly_enable)
export GAGE_PP_DATA=$(get_apps_defaults gage_pp_data)
export GAGE_PP_LOG=$(get_apps_defaults gage_pp_log)
export DB_NAME=$(get_apps_defaults db_name)
TEMP_DATE=`date -u +%m%d`
BUILD_HOURLY_LOGFILE=$GAGE_PP_LOG/build_hourly_$TEMP_DATE.log
if [[ $BUILD_HOURLY_ENABLE = OFF ]]
then
echo "Build hourly is not enabled...Exiting" >>$BUILD_HOURLY_LOGFILE
exit 1
fi
#Usage
USAGE="run_build_hourly lookback_window_in_hrs <end_time as \"YYYY-MM-DD HH:MM:SS\"> or run_build_hourly"
if [[ ! ( ($# -eq 2) || ($# -eq 0) ) ]]
then
print -u2 "\nUSAGE: $USAGE\n"
exit 1
fi
if [[ $# -eq 0 ]]
then
LOOKBACK_WINDOW=3 #default num of hours for which the hourly total is calculated
else
LOOKBACK_WINDOW=$1 #number of hours for which the hourly total is calculated
fi
# Command line arguments to the program build_hourly
END_TIME="$2"
DATABASE_NAME=$DB_NAME
MIN_PERCENT_FILLED=0.75 # Create the load file for the hour only if
# the data obtained by summation of the sub
# hourly data covers the whole hour
TYPE_HANDLING=PTYPE # SAME / PTYPE
# SAME - Retain the org type source
# PTYPE - Replace the R in RG, RZ,
# RR etc to P
echo "******************************" >>$BUILD_HOURLY_LOGFILE
echo "Started Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
# build_hourly -d dbname -l lookback_window -p percent_filled
# -t type_source [-e end_time]
runso rary.ohd.pproc build_hourly_main -d $DATABASE_NAME -l $LOOKBACK_WINDOW \
-p $MIN_PERCENT_FILLED -t $TYPE_HANDLING -e "$END_TIME" >>$BUILD_HOURLY_LOGFILE 2>&1
echo " " >>$BUILD_HOURLY_LOGFILE
echo "Stopped Build Hourly Script..." >>$BUILD_HOURLY_LOGFILE
echo "******************************" >>$BUILD_HOURLY_LOGFILE
integer CNTR=1
while [[ 1 -eq 1 ]]
do
NUM_OF_FILES=`ls -1 $GAGE_PP_DATA | grep BUILD | wc -l`
if [[ ($NUM_OF_FILES -ne 0) && ($CNTR -lt 15) ]]
then
sleep 2
CNTR=CNTR+1
else
break
fi
done

View file

@ -1,112 +0,0 @@
#! /bin/ksh
# set_hydro_env
#
# This script sets up the environment so that ohdlib based programs can be
# executed from the command line.
#
#AWIPS_MODIFICATION_BLOCK_BEGIN
# ****** Update This Line as Needed
export AW_SITE_IDENTIFIER=OAX
# locate EDEX_HOME
if [ "${EDEX_HOME}" = "" ]; then
default_edex_home="/awips2/edex"
if [ ! -d ${default_edex_home} ]; then
echo "Unable to locate EDEX_HOME: /awips2/edex"
exit 1
fi
export EDEX_HOME=${default_edex_home}
fi
if [ "${LOCALIZATION_ROOT}" = "" ]; then
export LOCALIZATION_ROOT=${EDEX_HOME}/data/utility/common_static
fi
if [ "${apps_dir}" = "" ]; then
export apps_dir=/awips2/edex/data/share/hydroapps
fi
# update the path to include the location of the runso rary.ohd.utility
export PATH=$PATH:${EDEX_HOME}/bin:${apps_dir}/bin
# contains the path to the ohd libraries and JNI-related library/libraries
export LD_LIBRARY_PATH=${EDEX_HOME}/lib/native/linux32:${apps_dir}/lib/native/linux32:$LD_LIBRARY_PATH
export RFCLX=${apps_dir}
#AWIPS_MODIFICATION_BLOCK_END
export POSTGRESQLDIR=/usr
export POSTGRESQLBINDIR=/awips2/psql/bin
export APPS_DEFAULTS_SITE=${LOCALIZATION_ROOT}/site/${AW_SITE_IDENTIFIER}/hydro/Apps_defaults
export APPS_DEFAULTS=${LOCALIZATION_ROOT}/base/hydro/Apps_defaults
# define a get_apps_defaults function for retrieving application token values
get_apps_defaults() {
runso rary.ohd.util gad $1
}
# OS_SUFFIX is the suffix used for filenames to identify the operating system
export OS_SUFFIX=".LX"
export geo_data=$RFCLX/geo_data
export PGOPTIONS='-c standard_conforming_strings=on'
export FXA_HOME=/awips/fxa
export FXA_BIN_DIR=$FXA_HOME/bin
# The following variables provide a JDBC URL or pieces thereof for
# SSHP and any other java programs that access the IHFS database
# The final form of the url is something like
# jdbc:postgresql://dx1:5432/hd_ob6abc?user=oper
export PGHOST=$(get_apps_defaults pghost)
export PGPORT=$(get_apps_defaults pgport)
export DB_NAME=$(get_apps_defaults db_name)
export PGUSER=$(get_apps_defaults pguser)
export DAMCAT_DB_NAME=$(get_apps_defaults damcat_db_name)
export JDBCURL=jdbc:postgresql://$PGHOST:$PGPORT/$DB_NAME?user=$PGUSER
export DAMCREST_JDBCURL=jdbc:postgresql://$PGHOST:$PGPORT/$DAMCAT_DB_NAME?user=$PGUSER
if [ "${PGSQL_DRIVER_DIR}" = "" ]; then
DB_DRIVER_DIR=${EDEX_HOME}/lib/dependencies/org.postgres
else
DB_DRIVER_DIR=${PGSQL_DRIVER_DIR}/
fi
cd $DB_DRIVER_DIR
files=$(find . | grep postgresql | grep jar)
for i in $files
do
echo $i
PGSQL_DRIVER_JAR=$i
done
export DB_DRIVER_PATH=${DB_DRIVER_DIR}/${PGSQL_DRIVER_JAR}
#===========================================================================
# This is stuff extracted from the original HYDRO_ENV
# and are used in run and start scripts and other files
#===========================================================================
# The following variable is needed for lots of scripts
export WHFS_BIN_DIR=$(get_apps_defaults whfs_bin_dir)
export WHFS_LOCAL_BIN_DIR=$(get_apps_defaults whfs_local_bin_dir)
#These variables needed to establish path to input and output files to the model in the DamCrest Application
export WHFS_LOCAL_DATA_DIR=$(get_apps_defaults whfs_local_data_dir)
export DAMCREST_DATA_DIR=$(get_apps_defaults damcrest_data_dir)
# this variable is needed to find gvim resource file .vimrc
export DAMCREST_RES_DIR=$(get_apps_defaults damcrest_res_dir)
# The following variable is needed for DamCat
export WFO_HELP_DIR=$(get_apps_defaults standard_data_dir)/help
# the following are needed for River/Precip Monitor
export RPF_LINE_WIDTH=$(get_apps_defaults rpf_linewidth)
export WHFS_UTIL_LOG_DIR=$(get_apps_defaults whfs_util_log_dir)
umask 000