xan/awips2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Issue #179

Browse source 
Former-commit-id: 6b3ebc4fef [formerly a0626cdc00] [formerly 6b3ebc4fef [formerly a0626cdc00] [formerly b23c6f59f7 [formerly d8f5c94fdc7a3332293e24c5e59d1d7177a5fcd9]]]
Former-commit-id: b23c6f59f7
Former-commit-id: 2955d5c9f0 [formerly 6d58d8c016]
Former-commit-id: 5ac5774552
This commit is contained in:
Brian.Dyke 2014-02-04 16:53:34 -05:00
parent d099529ef7
commit cc4a735533
8 changed files with 1902 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
edexOsgi/com.raytheon.uf.common.ohd/utility/common_static/base/hydro/Apps_defaults
View file

@ -204,7 +204,7 @@ server_name : ONLINE # Informix database server name
db_name : hd_ob92lwx # IHFS database name
damcat_db_name : dc_ob5xxx # Dam Catalog database name
hdb_db_name : ob81_histdata # Historical database.
pghost : localhost # The machine PostGres is running on
pghost : dx1f # The machine PostGres is running on
pguser : awips # The user allowed to access PostGres
pgport : 5432 # The PostGres Server port
adb_name : adb_ob7xxx # RFC archive database name
@ -1843,7 +1843,11 @@ dhm_d2d_notify_bin_dir : /awips/fxa/bin # d2d notify bin dir
rdhm_input_dir : $(geo_data)
dhm_rain_plus_melt_data_dir: $(geo_data)
# ================== end of SSHP Directory Structure tokens ========================
# nrldb tokens
nrldb_log : $(whfs_log_dir)/nrldb
nrldb_data : $(whfs_local_data_dir)/nrldb
nrldb_config : $(whfs_config_dir)/nrldb
nrldb_tmp : /awips/hydroapps/whfs/local/data/output
# The syntax needed in the file is:
#

38
nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.ksh Executable file
View file

@ -0,0 +1,38 @@
#!/usr/bin/ksh
###############################################################################
# AWIPS2 wrapper script for the daily scheduled NRLDB process. This uses the #
# nrldb.pl script to extract the static data from the IHFSDB, packages it in #
# an XML file and uploads it to the NRLDB server on the NHOR. #
# #
# Mark Armstrong (HSD) - 10/17/2013 #
###############################################################################
RUN_FROM_DIR=`dirname $0`
#echo "RFD: $RUN_FROM_DIR"
# set up SOME environment variables for WHFS applications
export PGSQL_DRIVER_DIR=/awips2/cave/plugins/org.postgres_9.2.0
export EDEX_HOME=/awips2/edex
export apps_dir=/awips2/edex/data/share/hydroapps
. $RUN_FROM_DIR/../../set_hydro_env
. $RUN_FROM_DIR/../../check_app_context
export NRLDB_DATA=$(get_apps_defaults nrldb_data)
#echo "NRLDB data: $NRLDB_DATA"
export NRLDB_LOG=$(get_apps_defaults nrldb_log)
#echo "NRLDB log: $NRLDB_LOG"
export NRLDB_CONFIG=$(get_apps_defaults nrldb_config)
#echo "NRLDB config: $NRLDB_CONFIG"
export WHFS_BIN=$(get_apps_defaults whfs_bin_dir)
#echo "WHFS_BIN: $WHFS_BIN"
export NRLDBLOGFILE=${NRLDB_LOG}/nrldb.log
export NRLDBTMPFILE=${NRLDB_LOG}/nrldb.tmp
tail -5000 $NRLDBLOGFILE > $NRLDBTMPFILE
mv $NRLDBTMPFILE $NRLDBLOGFILE
${WHFS_BIN}/nrldb.pl -t wfo -u
#

1415
nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.pl Executable file
View file

File diff suppressed because it is too large Load diff

21
nativeLib/files.native/awipsShare/hydroapps/whfs/bin/run_update_nrldb.ksh Executable file
View file

@ -0,0 +1,21 @@
#!/bin/ksh
RUN_FROM_DIR=`dirname $0`
# set up SOME environment variables for WHFS applications
export PGSQL_DRIVER_DIR=/awips2/cave/plugins/org.postgres_9.2.0
. $RUN_FROM_DIR/../../set_hydro_env
. $RUN_FROM_DIR/../../check_app_context
export APPS_DEFAULTS=~/caveData/common/base/hydro/Apps_defaults
export APPS_DEFAULTS_SITE=~/caveData/common/site/${AW_SITE_IDENTIFIER}/hydro/Apps_defaults
export PGUSER="awips"
export BIN_DIR=`get_apps_defaults whfs_bin_dir"`
export NRLDB_LOG=`get_apps_defaults nrldb_log`
export NRLDB_CONFIG=`get_apps_defaults nrldb_config`
export NRLDB_DATA=`get_apps_defaults nrldb_data`
#export NRLDB_TMP=`get_apps_defaults nrldb_tmp`
export db_name=`get_apps_defaults db_name`
export PGHOST=`get_apps_defaults pghost`
$RUN_FROM_DIR/update_nrldb.pl
exit 0

167
nativeLib/files.native/awipsShare/hydroapps/whfs/bin/send_nrldb_update.sh Executable file
View file

@ -0,0 +1,167 @@
#!/bin/sh
###############################################################################
# This script is run at the field office to send ad-hoc updates to the NRLDB
# server, then on to the AHPS CMS. It can be run at any time. It is designed
# to send small, time-sensitive updates to the CMS. It takes two argument
# lists:-table table names (comma-separated) and -lid lid names
# (comma-separated). It parses the arguments, selects the updated data from
# the database and builds an SQL formatted text file for use on the nrldb and
# CMS databases. The SQL file contains a delete staement that deletes the
# pre-existing data for the lid/table combinations, before running the inserts
#
# Usage: send_nrldb_update.sh -table <table1>,<table2>,... -lid <lid1>,<lid2>,...
# Example: send_nrldb_update.sh -table rating,floodstmt -lid BRKM2,CBEM2
#
if [ $# -ne 4 ]
then
echo "Incorrect number of arguments entered: $#"
echo "Correct Arguments are:"
echo "send_nrldb_update.sh -table table1,table2 -lid lid1,lid2"
echo "Any number of tables and lids may be specified, but they need to be in a comma separated list with no spaces between commas and table/lid names"
exit 0
fi
# set up SOME environment variables for NRLDB applications
# get the nrldb host and wfo from the nrldb.conf file/database
nrldb_host=`grep nrldb_host $NRLDB_CONFIG/nrldb.conf | cut -d= -f2 | sed 's/"//g' | sed 's/ //g'`
echo "DB NAME: $db_name"
wfo=`psql -h $PGHOST -d $db_name -c "select hsa from admin;" | tail -3 | head -1 | sed -e 's/ //g'`
echo `date`
echo "WFO $wfo"
# create the final SQL file that will be sent to the NRLDB host
timestamp=`date +%Y%m%d%H%N`
sql_file="${wfo}_update_${timestamp}.sql"
if [ -f $sql_file ]
then
rm $sql_file
fi
# build the list of tables/lids to send
lid_list="XXXXX"
table_list="XXXXX"
while [ $# -gt 0 ]
do
case "$1" in
-lid) lid_list="$2,";shift;;
-table) table_list="$2,";shift;;
*) break;;
esac
shift
done
# set the last update information for update_nrldb.pl to use
echo `date` > ${NRLDB_LOG}/last_nrldb_update.txt
up_lid_list=`echo $lid_list | sed 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'`
echo "lid list: $up_lid_list" >> ${NRLDB_LOG}/last_nrldb_update.txt
echo "table_list: $table_list" >> ${NRLDB_LOG}/last_nrldb_update.txt
#loop through the tables/lids
if [ $table_list != "XXXXX" ]
then
pos=1
table="XXXXX"
ltable=`echo $table | wc -m`
while [ $ltable -gt 4 ]
do
table=`echo $table_list | cut -d"," -f$pos`
pos=`expr $pos + 1`
ltable=`echo $table | wc -m`
if [ $ltable -gt 4 ]
then
lid="XXXXX"
lpos=1
llid=`echo $lid | wc -m`
while [ $llid -gt 3 ]
do
lid=`echo $up_lid_list | cut -d"," -f$lpos`
lpos=`expr $lpos + 1`
llid=`echo $lid | wc -m`
if [ $llid -gt 3 ]
then
# fetch the values from the DB and edit them
export PGUSER=awips
touch $NRLDB_TMP/update.txt
chmod ugo+rw $NRLDB_TMP/update.txt
ls -l $NRLDB_TMP/update.txt
psql -h $PGHOST -d $db_name -c "copy (select * from $table where lid = '$lid') to '$NRLDB_TMP/update.txt' with delimiter '|';"
cp $NRLDB_TMP/update.txt ${NRLDB_DATA}/update.txt
sed -f ${NRLDB_CONFIG}/sed_script.txt ${NRLDB_TMP}/update.txt > ${NRLDB_DATA}/update11.txt
sed -e "s/|/'|'/g" ${NRLDB_DATA}/update11.txt > ${NRLDB_DATA}/update1.txt
sed -e "s/^/insert into $table values('/g" ${NRLDB_DATA}/update1.txt > ${NRLDB_DATA}/update2.txt
sed -e "s/$/');/g" ${NRLDB_DATA}/update2.txt > ${NRLDB_DATA}/update3.txt
sed -e "s/|/,/g" ${NRLDB_DATA}/update3.txt > ${NRLDB_DATA}/update4.txt
if [ -f "${NRLDB_DATA}/update.txt" ]
then
update_lines=`wc -l "${NRLDB_DATA}/update.txt" | cut -d" " -f1`
else
echo "No update file found".
update_lines=0
fi
if [ $update_lines -gt 0 ]
then
if [ $table != "location" -a $table != "riverstat" ]
then
echo "delete from $table where lid = '$lid';" >> ${NRLDB_DATA}/$sql_file
fi
cat ${NRLDB_DATA}/update4.txt >> ${NRLDB_DATA}/$sql_file
fi
# location and riverstat require a special forecast since they have dependent tables via foreign keys
if [ $table = "location" ]
then
sql_stmt="update location set lid = '$lid'"
for col in county coe cpm detail elev hdatum hsa hu lat lon lremark lrevise name network rb rfc sbd sn state waro wfo wsfo type des det post stntype tzone
do
psql -h $PGHOST -d $db_name -c "select $col from location where lid = '$lid' and $col is not null;" > ${NRLDB_DATA}/update.txt
ct_zero=`grep -c "0 row" ${NRLDB_DATA}/update.txt`
if [ $ct_zero -eq 0 ]
then
export val=`cat ${NRLDB_DATA}/update.txt | head -3 | tail -1 | cut -c2-80`
new_val=`echo "$val" | sed -f ${NRLDB_CONFIG}/sed_script.txt`
sql_stmt="$sql_stmt, $col = '$new_val'"
fi
done
sql_stmt="$sql_stmt where lid = '$lid';"
echo $sql_stmt >> ${NRLDB_DATA}/$sql_file
elif [ $table = "riverstat" ]
then
sql_stmt="update riverstat set lid = '$lid'"
for col in primary_pe bf cb da response_time threshold_runoff fq fs gsno level mile pool por rated lat lon remark rrevise rsource stream tide backwater vdatum action_flow wstg zd ratedat usgs_ratenum uhgdur use_latest_fcst
do
psql -h $PGHOST -d $db_name -c "select $col from riverstat where lid = '$lid' and $col is not null;" > ${NRLDB_DATA}/update.txt
ct_zero=`grep -c "0 row" ${NRLDB_DATA}/update.txt`
if [ $ct_zero -eq 0 ]
then
export val=`cat ${NRLDB_DATA}/update.txt | head -3 | tail -1 | cut -c2-80`
new_val=`echo "$val" | sed -f ${NRLDB_CONFIG}/sed_script.txt`
sql_stmt="$sql_stmt, $col = '$new_val'"
fi
done
sql_stmt="$sql_stmt where lid = '$lid';"
echo $sql_stmt >> ${NRLDB_DATA}/$sql_file
fi
fi
done
fi
done
# send the SQL file to the NRLDB server
if [ -f ${NRLDB_DATA}/$sql_file ]
then
rsync -av ${NRLDB_DATA}/$sql_file ${nrldb_host}\::nrldb_update/
echo "SQL file: $sql_file created for lids: $up_lid_list and tables: $table_list"
else
echo "No SQL file created. Database contained no entries for lids: $up_lid_list and tables: $table_list"
fi
fi
# remove the temp files to keep the directory clean
for temp_file in ${NRLDB_DATA}/update.txt ${NRLDB_DATA}/update11.txt ${NRLDB_DATA}/update1.txt ${NRLDB_DATA}/update2.txt ${NRLDB_DATA}/update3.txt ${NRLDB_DATA}/update4.txt
do
if [ -f $temp_file ]
then
rm $temp_file
fi
done

248
nativeLib/files.native/awipsShare/hydroapps/whfs/bin/update_nrldb.pl Executable file
View file

@ -0,0 +1,248 @@
#!/usr/bin/perl
################################################################################
# update_nrldb.pl is the GUI for the Ad-Hoc update process. ## This process was put in place so that WFOs could update information #
# between daily runs of the NRLDB update process. The information is #
# collected at the WFO, sent to the NRLDB central server and then forwarded to #
# CMS servers outside of the AWIPS firewall. #
# #
# Developer: Mark Armstrong (OCWWS/HSD) #
# Developed 2011 - Modified for AWIPS2 2013 #
################################################################################
use Tk;
use strict;
use warnings;
use AppConfig qw(:expand :argcount);
use DBI;
our $BIN_DIR = `echo \$BIN_DIR`;
chomp($BIN_DIR);
our $NRLDB_LOG = `echo \$NRLDB_LOG`;
chomp($NRLDB_LOG);
my $lids;
my $tables;
# Set up some inial configuration. Most of this comes from the hydroGen input file: hg.cfg
$ENV{HYDROGENHOME} = "/awips/hydroapps/HydroGen" if ! defined $ENV{HYDROGENHOME};
my %cfg = ( DEBUG => 0, # debug mode on or off
PEDANTIC => 0, # be patient with warnings/errors
CREATE => 1, # create variables, defining not required...
GLOBAL => { # for all config options unless overridden...
EXPAND => EXPAND_ALL, # expand ~, $ENV{*}, and $(var)
ARGCOUNT => ARGCOUNT_ONE, # each config expects an arg unless overriden...
ARGS => '=s' # each arg is a string unless overriden
}
);
my $config = AppConfig->new(\%cfg); # create config object
$config->define('version',{ ALIAS => 'V',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0});
$config->define('help',{ ALIAS => 'h',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0});
$config->define('man',{ ALIAS => 'm',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0});
$config->define('DBengine',{ VALIDATE => '[\w]+',DEFAULT => "Pg"});
$config->define('DBname',{ VALIDATE => '[\w]+',DEFAULT => "hd_ob8xxx"});
$config->define('DBhost',{ VALIDATE => '[-\w]+',DEFAULT => "dx1f"});
$config->define('DBport',{ ARGS => '=i',DEFAULT => 5432});
$config->define('master',{ VALIDATE => '[.\w]+',DEFAULT => "HGstation"});
$config->define('basedir',{ VALIDATE => '[- /.\w]+',DEFAULT => $ENV{HYDROGENHOME} . "/bin"});
$config->file($ENV{HYDROGENHOME} . "/input/hg.cfg"); # look in user's $HYDROGENHOME to find configured settings
$config->args(\@ARGV); # get config settings from the command-line, overwriting any settings from the file...
my $master = $config->get('master'); # name of DB table or view which holds master list of IDs for which MXD files are to be generated...
my $DBengine = $config->get('DBengine');
my $DBname = $config->get('DBname');
my $DBhost = $config->get('DBhost');
my $DBport = $config->get('DBport');
my $baseDir = `pwd`;
chomp $baseDir;
my $DBstr;
my $wildcard;
#Open a database connection and get the list of LIDs from the IHFS DB
if($DBengine eq "Pg") {
$DBstr = "dbi:$DBengine:dbname=$DBname;host=$DBhost;port=$DBport";
$wildcard = '%';
} else {
$DBstr = "dbi:$DBengine:$DBname";
$wildcard = '*';
}
my $dbh = DBI->connect("$DBstr",undef,undef,{ChopBlanks => 1}) or warn $DBI::errstr;
# creates the list of WFOs based on the HydroGen .xxx_backup files
# and builds the query to create the list of LIDs
my $wfo=`ls -a /awips/hydroapps/HydroGen/ | grep _backup | cut -c2-4`;
my $list_len=length $wfo;
my $num_wfos=$list_len/4;
my $index=1;
my $off=0;
my $wfoid=substr($wfo,$off,3);
my $wfoID=uc $wfoid;
my $wfo_query = "(location.hsa = \'$wfoID\'";
while ($index < $num_wfos){
$off+=4;
$wfoid=substr($wfo,$off,3);
$wfoID=uc $wfoid;
$wfo_query .= " or location.hsa = \'$wfoID\'";
$index++;
}
$wfo_query .= ")";
#my $list_type="river";
our $mw = MainWindow->new;
$mw->title('Ad-Hoc NRLDB Update');
my $lst_lab= $mw->Label(-text => 'Add any Unlisted Locations (comma-separated): ');
my $sql = "select distinct hgstation.lid,location.name,location.hsa from hgstation,location where hgstation.lid = location.lid and $wfo_query order by 3,1;";
# get the list of LIDs
my $qhw = $dbh->prepare("$sql") or warn $DBI::errstr;
our @lid_list; # = ($wildcard);
#get the data from the DB
get_results($qhw,\@lid_list);
#set up a static array with the tables that are allowed for ad-hoc updates
#table_list is the actual name of the DB tables, while tabledesc is a friendlier description that is displayed to the user
our @table_list = ('location','riverstat','crest','floodstmt','hgstation','floodcat','lowwater');
my @tabledesc = ('Location','Riverstat','Crest History','Impacts','HGstation','Flood Categories','Low Water');
$dbh->disconnect();
#manipulate the results of the lid/hsa/name query for better display
my @liddeschsa;
our @lidsend;
$index=0;
my $num_lids=scalar(@lid_list);
while ($index < $num_lids){
my $line = $lid_list[$index];
my @results = split('\|',$line);
#my $lid = $lid_list[$index];
my $lid_lid = $results[0];
my $lid_name = $results[1];
my $lid_hsa = $results[2];
push(@liddeschsa,"$lid_hsa | $lid_lid | $lid_name");
push(@lidsend,$lid_lid);
$index++;
}
# Create the GUI object
# Labels for the LID and table scroll boxes
my $misc_ent = $mw->Entry();
my $label1 = $mw->Label(-text => 'HSA|LID|Location Name');
my $label2 = $mw->Label(-text => 'Tables');
# Create the scroll boxes for the LIDs and tables
my $lb1 = $mw->Scrolled('Listbox',
-scrollbars => 'osoe',-width=>50,
-selectmode => 'multiple', -exportselection=>0);
my $lb2 = $mw->Scrolled('Listbox',
-scrollbars => 'osow',-width=>20,
-selectmode => 'multiple',-exportselection=>0);
# Add the arrays that we want to display in the list boxes
$lb1->insert('end', @liddeschsa);
$lb2->insert('end', @tabledesc);
# Create the buttons
my $exit = $mw->Button(-text => 'Exit',
-command => [$mw => 'destroy']);
my $send = $mw->Button(-text => 'Send',
-command => \&send_button);
my $show_log = $mw->Button(-text => 'Show Log',
-command => \&show_log);
my $update_list = $mw->Button(-text => 'Update List', -command => \&upd_list);
# create the label and text box for the last pdate window
my $status_box = $mw->Text(-width=>20, -height=>3);
my $lb_status = $mw->Label(-width=>20, -height=>3,-text=>"Last Ad-Hoc Update:");
my $last_update = `cat $NRLDB_LOG/last_nrldb_update.txt`;
$status_box->insert('end',"$last_update");
# Crate the GUI using grid to specify the physical locations of the objects
$label1->grid(-row=>1, -column=>1, -columnspan=>3) ;
$label2->grid(-row=>1, -column=>4) ;
$lb1->grid(-row=>2, -column=>1, -columnspan=>3, -sticky=>"ew") ;#pack;
$lb2->grid(-row=>2, -column=>4, -columnspan=>1, -sticky=>"w") ;#pack;
$lst_lab->grid(-row=>3, -column=>1, -columnspan=>1);
$misc_ent->grid(-row=>3, -column=>2);
$lb_status->grid(-row=>4, -column=>1);
$status_box->grid(-row=>4, -column=>2, -columnspan=>3, -sticky=>"ew");
$send->grid(-row=>5, -column=>1) ;#pack;
$show_log->grid(-row=>5,-column=>2);
$exit->grid(-row=>5, -column=>4) ;#pack;
MainLoop;
# End of main
#
# The Send button functionality function
sub send_button {
# Get the indices of the selected array items
my @LIDindex = $lb1->curselection;
my @Tableindex = $lb2->curselection;
my $index=1;
my $misc_lid = $misc_ent-> get();
# build the lists of LIDs and tables
$tables = $table_list[$Tableindex[0]];
my $numLIDs=@LIDindex;
print "numLIDs: $numLIDs\n";
my $numTables=@Tableindex;
if ($numLIDs > 0){
$lids = $lidsend[$LIDindex[0]];
while ($index < $numLIDs){
$lids .= "," . $lidsend[$LIDindex[$index]];
$index++;
}
$lids .= "," . $misc_lid;
} else {
$lids=$misc_lid;
}
$index=1;
while ($index < $numTables){
$tables .= "," . $table_list[$Tableindex[$index]];
$index++;
}
# Create the call to the script and execute it using system()
my $cmd = "${BIN_DIR}/send_nrldb_update.sh -table $tables -lid $lids > ${NRLDB_LOG}/send_nrldb_update.log\n";
system($cmd);
# Create a dialog box to inform the user that their data has been sent
my $dsend=$mw->Dialog(-title=>'Sent NRLDB Update',-buttons=>['OK']);
my $text_field="NRLDB Update Sent for LIDs: $lids \n and tables: $tables\n";
my $box=$dsend->add('Label',-text=>"$text_field")->pack(-side => 'left',-fill => 'both',-expand => 1);
my $button = $dsend->Show;
}
# This subroutine, copied from Mark Fenbers bless program, takes a db query and returns an array of results
sub get_results
{
my $qh = shift;
my $array = shift;
my $record;
if(defined $qh) {
if($qh->execute(@_)) {
while($record = $qh->fetchrow_arrayref) {
foreach (@$record) { $_ = "" if ! defined $_; }
push @$array,(join '|',@$record);
}
} else {
warn $DBI::errstr;
}
} else { warn "unable to prepare query \"$sql\"\n"; }
}
#This subroutine displays the log from the send script in the form of a dialog box
sub show_log
{
use Tk::Dialog;
my $text_field=`cat ${NRLDB_LOG}/send_nrldb_update.log`;
my $d = $mw->Dialog(-title=>'Show Log',-buttons => ['OK']);
my $box=$d->add('Label',-text=>"$text_field")->pack(-side => 'left',-fill => 'both',-expand => 1);
my $button = $d->Show;
}

6
nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/nrldb.conf.baseline Executable file
View file

@ -0,0 +1,6 @@
dbhost = "dx1f"
dbuser = "awips"
dbpass = ""
nrldb_host = "165.92.28.1"
site = "CCC"
dbname = "hd_ob92ccc"

1
nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/sed_script.txt Normal file
View file

@ -0,0 +1 @@
s/'/\\'/g