From 6d58d8c01686d56d8512f4af1230d1c4f9df3712 Mon Sep 17 00:00:00 2001 From: "Brian.Dyke" Date: Tue, 4 Feb 2014 16:53:34 -0500 Subject: [PATCH] Issue #179 Former-commit-id: a0626cdc00eb9ca7b08795631d50090c42d9029d [formerly d8f5c94fdc7a3332293e24c5e59d1d7177a5fcd9] Former-commit-id: b23c6f59f767c181c6435802c08ccb751ad0dd2d --- .../common_static/base/hydro/Apps_defaults | 8 +- .../awipsShare/hydroapps/whfs/bin/nrldb.ksh | 38 + .../awipsShare/hydroapps/whfs/bin/nrldb.pl | 1415 +++++++++++++++++ .../hydroapps/whfs/bin/run_update_nrldb.ksh | 21 + .../hydroapps/whfs/bin/send_nrldb_update.sh | 167 ++ .../hydroapps/whfs/bin/update_nrldb.pl | 248 +++ .../local/data/app/nrldb/nrldb.conf.baseline | 6 + .../whfs/local/data/app/nrldb/sed_script.txt | 1 + 8 files changed, 1902 insertions(+), 2 deletions(-) create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.ksh create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.pl create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/bin/run_update_nrldb.ksh create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/bin/send_nrldb_update.sh create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/bin/update_nrldb.pl create mode 100755 nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/nrldb.conf.baseline create mode 100644 nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/sed_script.txt diff --git a/edexOsgi/com.raytheon.uf.common.ohd/utility/common_static/base/hydro/Apps_defaults b/edexOsgi/com.raytheon.uf.common.ohd/utility/common_static/base/hydro/Apps_defaults index d3908a8b55..94dcef9616 100644 --- a/edexOsgi/com.raytheon.uf.common.ohd/utility/common_static/base/hydro/Apps_defaults +++ b/edexOsgi/com.raytheon.uf.common.ohd/utility/common_static/base/hydro/Apps_defaults @@ -204,7 +204,7 @@ server_name : ONLINE # Informix database server name db_name : hd_ob92lwx # IHFS database name damcat_db_name : dc_ob5xxx # Dam Catalog database name hdb_db_name : ob81_histdata # Historical database. -pghost : localhost # The machine PostGres is running on +pghost : dx1f # The machine PostGres is running on pguser : awips # The user allowed to access PostGres pgport : 5432 # The PostGres Server port adb_name : adb_ob7xxx # RFC archive database name @@ -1843,7 +1843,11 @@ dhm_d2d_notify_bin_dir : /awips/fxa/bin # d2d notify bin dir rdhm_input_dir : $(geo_data) dhm_rain_plus_melt_data_dir: $(geo_data) # ================== end of SSHP Directory Structure tokens ======================== - +# nrldb tokens +nrldb_log : $(whfs_log_dir)/nrldb +nrldb_data : $(whfs_local_data_dir)/nrldb +nrldb_config : $(whfs_config_dir)/nrldb +nrldb_tmp : /awips/hydroapps/whfs/local/data/output # The syntax needed in the file is: # diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.ksh b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.ksh new file mode 100755 index 0000000000..9b0f0198fe --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.ksh @@ -0,0 +1,38 @@ +#!/usr/bin/ksh +############################################################################### +# AWIPS2 wrapper script for the daily scheduled NRLDB process. This uses the # +# nrldb.pl script to extract the static data from the IHFSDB, packages it in # +# an XML file and uploads it to the NRLDB server on the NHOR. # +# # +# Mark Armstrong (HSD) - 10/17/2013 # +############################################################################### +RUN_FROM_DIR=`dirname $0` +#echo "RFD: $RUN_FROM_DIR" +# set up SOME environment variables for WHFS applications +export PGSQL_DRIVER_DIR=/awips2/cave/plugins/org.postgres_9.2.0 +export EDEX_HOME=/awips2/edex +export apps_dir=/awips2/edex/data/share/hydroapps +. $RUN_FROM_DIR/../../set_hydro_env +. $RUN_FROM_DIR/../../check_app_context + + +export NRLDB_DATA=$(get_apps_defaults nrldb_data) +#echo "NRLDB data: $NRLDB_DATA" + +export NRLDB_LOG=$(get_apps_defaults nrldb_log) +#echo "NRLDB log: $NRLDB_LOG" + +export NRLDB_CONFIG=$(get_apps_defaults nrldb_config) +#echo "NRLDB config: $NRLDB_CONFIG" + +export WHFS_BIN=$(get_apps_defaults whfs_bin_dir) +#echo "WHFS_BIN: $WHFS_BIN" + +export NRLDBLOGFILE=${NRLDB_LOG}/nrldb.log +export NRLDBTMPFILE=${NRLDB_LOG}/nrldb.tmp +tail -5000 $NRLDBLOGFILE > $NRLDBTMPFILE +mv $NRLDBTMPFILE $NRLDBLOGFILE + +${WHFS_BIN}/nrldb.pl -t wfo -u + +# diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.pl b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.pl new file mode 100755 index 0000000000..cc1d5ab6d5 --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/nrldb.pl @@ -0,0 +1,1415 @@ +#!/usr/bin/perl + +use strict; +use DBI; +use AppConfig qw(:expand :argcount); + + +#Set/define command line args +my %cfg = ( DEBUG => 0); # debug mode on or off +my $config = AppConfig->new(\%cfg); # create config object +$config->define('type',{ARGCOUNT => ARGCOUNT_ONE, VALIDATE => '(WFO|RFC|HQ|wfo|rfc|hq)', ALIAS => 'T'}); +$config->define('local-control-file',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'L',DEFAULT => 0}); +$config->define('upload',{ARGCOUNT => ARGCOUNT_NONE, ALIAS => 'U', DEFAULT => 0}); +$config->define('wfo-id',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'W', DEFAULT => 0}); +$config->define('rfc-id',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'R', DEFAULT => 0}); +$config->define('out-xmlfile',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'O', DEFAULT => 0}); +$config->define('input-xmlfile',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'I', DEFAULT => 0}); +$config->define('check',{ARGCOUNT => ARGCOUNT_NONE, ALIAS => 'C', DEFAULT => 0}); +$config->define('verbose',{ARGCOUNT => ARGCOUNT_NONE, ALIAS => 'V', DEFAULT => 0}); +$config->define('dbname',{ARGCOUNT => ARGCOUNT_ONE, ALIAS => 'D', DEFAULT => 0}); +$config->define('extract',{ARGCOUNT => ARGCOUNT_NONE, ALIAS => 'E', DEFAULT => 0}); +$config->define('delete',{ARGCOUNT => ARGCOUNT_NONE, ALIAS => 'A', DEFAULT => 0}); +$config->getopt(\@ARGV); + +our $type = uc($config->get('type')); +our $localControlFile = $config->get('local-control-file'); +our $Upload = $config->get('upload'); +our $wfoID = uc($config->get('wfo-id')); +our $rfcID = uc($config->get('rfc-id')); +our $outFile = $config->get('out-xmlfile'); +our $inFile = $config->get('input-xmlfile'); +our $check = $config->get('check'); +our $verbose = $config->get('verbose'); +our $dbname_flag = $config->get('dbname'); +our $extract = $config->get('extract'); +our $delete = $config->get('delete'); +our $office; +our $update_count = 0; +our $insert_count = 0; +our $error_count = 0; +our $total_count = 0; +our $file_name; +our $conf_dir; +my ($dbname, $host, $user, $pass, $nrldb_host, $backup_host); +my @delete_list; +my $delete_listRef; +print "db name flag: $dbname_flag\n"; +if($check) { + warn "-----Starting NRLDB installation check-----\nInstallation Complete.\n"; + print "Installation Complete.\n"; + exit 0; +} + + +#Get config file info +($dbname, $host, $user, $pass, $nrldb_host, $office, $backup_host) = read_config_file(); + +if(!$dbname_flag) +{ + if( -e "/awips/hydroapps/public/bin/get_apps_defaults") + { + $dbname = `/awips/hydroapps/public/bin/get_apps_defaults.LX db_name`; + } +} +else{ + $dbname = $dbname_flag; +} +# Do parameter checks +if($type eq "") +{ + print "No office type specified.\nusage: --type WFO|RFC|HQ\n\n"; + exit 1; +} +if($type eq "HQ") +{ + if($inFile eq 0) + { + print "No xml input file specified.\nusage: --type HQ --input-xmlfile 'file'\n\n"; + exit 1; + } + if($rfcID eq 0 && $wfoID eq 0) + { + print "You must specify a WFO/RFC office identifier with the HQ type.\n"; + exit 1; + } + + unless($rfcID eq 0) { + $office = $rfcID; + } + unless($wfoID eq 0) { + $office = $wfoID; + } + +} + +if($type eq "RFC") +{ + if($rfcID eq 0) + { + print "You must specify an RFC office identifier with the rfc option.\nusage: --type RFC --rfc-id IDRFC\n\n"; + exit 1; + } +} + + +#Connect to database +our $db = db_connect($dbname, $host, $user, $pass); + +my $date = getdate(); +print "---Starting NRLDB process at $office\, running as $type\---\n---$date\n\n" if($verbose); +warn "---Starting NRLDB process at $office\, running as $type\---\n---$date\n\n"; +print "Connected to database: $dbname\n" if($verbose); +warn "Connected to database: $dbname\n"; +#Determine what type of office is running nrldb software +if(($type eq "WFO") | ($type eq "RFC")) +{ + if($localControlFile eq 0) + { + download_control_file($type); + } + create_xml(); + if($Upload) + { + upload_xml($nrldb_host); + upload_xml($backup_host); + } +} +elsif($type eq "HQ") +{ + if($delete) + { + $delete_listRef = get_delete_list(); + @delete_list = @$delete_listRef; + foreach my $delete_table (@delete_list) + { + deleteValues($delete_table); + } + } + xml_parse(); +} + +print "\n-----------------------------\n\n" if($verbose); +warn "\n-----------------------------\n\n"; +exit 0; + + +# sub 'create_xml' is responsible for querying the database and putting the info into xml format. +sub create_xml +{ + +my $table_name; +my ($select_string, $field_string); +my $xml_string; +my $record_count; +my ($st, $at); +my $table_query; +my $query_error_flag; +my $numrows; +my $lid_flag; +my $pkey; +my ($pk_name, $field_name); +my $row; +my $extract_detail; +my %infohash; +my @tables; +my @fields; +my @fields_all; +my @select_array; +my @PK; +my @keys; +my (@pk_output, @fields_output); + +#read control file and put specified fields into array +my ($tables_ref, $fields_all_ref) = read_control_file(); +@tables = @$tables_ref; +@fields_all = @$fields_all_ref; + + $extract_detail = ''; +# print "EXTRACT: $extract\n"; + unless($extract eq 0) + { + $extract_detail = extract_detail(); + } + +# Start creating xml +$xml_string = "\n\n"; +foreach $table_name (@tables) +{ + + print "TABLE: $table_name\n" if($verbose); + warn "TABLE: $table_name\n"; + $select_string = ""; + $lid_flag = 1; + # Get primary key list for specified tables + @keys = $db->primary_key(undef, undef, $table_name); + + foreach $pkey (@keys) + { + # The following 6 lines were by mark Armstrong (HSD) on 2/26/09 + # to remove the quotes from primary keys. + # When primary keys occurred with quotes, the update queries + # were not successful. + if ($pkey =~ /"/){ + my $length_pkey = length $pkey; + $length_pkey -= 2; + my $new_pkey = substr($pkey,1,$length_pkey); + $pkey=$new_pkey; + } + push(@PK, "$table_name.$pkey"); + } + + @pk_output = grep(/$table_name\.\w*/, @PK); + print "\tPK: @pk_output\n" if($verbose); + warn "\tPK: @pk_output\n"; + @fields_output = grep(/$table_name\.\w*/, @fields_all); + print "\tFIELDS: @fields_output\n" if($verbose); + warn "\tFIELDS: @fields_output\n"; + + my $pk_count = @pk_output; + if($pk_count == 0) + { + print "No Primary Keys found for Table: $table_name\nContinuing\n\n" if($verbose); + warn "No Primary Keys found for Table: $table_name\nContinuing\n\n"; + next; + } + + #loop through arrays and put together a select string for specified table + foreach my $pk (@pk_output) + { + if($pk =~ /$table_name\.\w*/) + { + if($select_string eq "") + { + $select_string = "$pk"; + } + else + { + $select_string .= ",$pk"; + } + } + } + + + foreach my $fields (@fields_output) + { + if($select_string =~ /.*$fields.*/) + { + if($field_string eq "") + { + $field_string = "$fields"; + } + else + { + $field_string .= ",$fields"; + } + next; + } + elsif($fields =~ /.*ALL.*/) + { + $select_string = "*"; + last; + } + else + { + if($field_string eq "") + { + $field_string = "$fields"; + } + else + { + $field_string .= ",$fields"; + } + $select_string .= ",$fields"; + } + } + + + #print select string to be used + print "\n" if($verbose); + warn "\n"; + $query_error_flag = 0; + #if select string equal 'ALL' get a list of all fields in specified table by querying database info tables. + if($select_string eq "*") + { + + my $query_column1 = "SELECT c.oid + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE pg_catalog.pg_table_is_visible(c.oid) + AND c.relname ~ '^$table_name\$'"; + + my $attribute_query = "SELECT a.attname + FROM pg_catalog.pg_attribute a + WHERE a.attnum > 0 AND NOT a.attisdropped + AND a.attrelid = ($query_column1) + ORDER BY a.attnum;"; + + eval + { + $at = $db->prepare($attribute_query); + $at->execute() or die "Cannot execute: ".$at->errstr(); + }; + if($@) + {print "$@\n" if($verbose); warn "$@\n";} + + my $att_count = 0; + while ( defined ( my $attribues = $at->fetchrow_arrayref() ) ) + { + if($att_count > 0) + { + $select_string .= ",$table_name.@$attribues[0]"; + } + else + { + $select_string = "$table_name.@$attribues[0]"; + } + $att_count++; + } + $field_string = $select_string; + } + + #Check for lid in table + if($select_string !~ /$table_name\.lid/) + { + $lid_flag = lid_check($table_name); + } + + # Determine query depending on office type and other parameters + ## Revised query to properly select only counties from primary HSA or identified WFO - Ernie Wells February 09 ## + if($type eq "WFO") + { + if($wfoID eq 0) { + if($table_name =~ /location/) + { + $table_query = "SELECT $select_string FROM location, admin WHERE location.hsa = admin.hsa $extract_detail ORDER BY lid;"; + } elsif($table_name =~ /counties/) { + $table_query = "SELECT $select_string FROM counties, admin WHERE counties.wfo = admin.hsa;"; + } elsif($table_name =~ /rpffcstgroup/) { + $table_query = "SELECT distinct $select_string from rpffcstgroup join rpffcstpoint rp on rp.group_id = rpffcstgroup.group_id join location l on l.lid = rp.lid join admin on l.hsa = admin.hsa;"; + } elsif($table_name =~ /vtecevent/) { + $table_query = "SELECT $select_string FROM vtecevent WHERE vtecevent.geoid in (select location.lid from location, admin where location.hsa = admin.hsa) $extract_detail;"; + } elsif($table_name eq "height" || $table_name =~ /temperature/ || $table_name =~ /curpp/ || $table_name =~ /curpc/ || $table_name eq "discharge"){ + my $cutoff_dtime = getcutoffdate(); + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location, admin WHERE location.lid = $table_name.lid AND location.hsa = admin.hsa) and obstime > '$cutoff_dtime' $extract_detail ORDER BY lid;"; + } elsif($table_name =~ /fcstheight/ || $table_name =~ /fcstdischarge/) { + my $cutoff_dtime = getcutoffdate(); + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location, admin WHERE location.lid = $table_name.lid AND location.hsa = admin.hsa) and basistime > '$cutoff_dtime' $extract_detail ORDER BY lid;"; + } elsif($lid_flag == 1){ + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location, admin WHERE location.lid = $table_name.lid AND location.hsa = admin.hsa) $extract_detail ORDER BY lid;"; + } + else { + $table_query = "SELECT $select_string FROM $table_name\;"; + } + } + else { + if($table_name =~ /location/) + { + if($extract eq 0) { + $table_query = "SELECT $select_string FROM location WHERE location.hsa = '$wfoID' $extract_detail ORDER BY lid;"; + } else { + $table_query = "SELECT $select_string FROM location WHERE location.hsa like '%' $extract_detail ORDER BY lid;"; + } + } elsif($table_name =~ /counties/) { + if($extract eq 0) { + $table_query = "SELECT $select_string FROM counties WHERE counties.wfo = '$wfoID';"; + } else { + $table_query = "SELECT $select_string FROM counties WHERE counties.wfo in (select hsa from location where hsa is not null $extract_detail) ;"; + } + } elsif($table_name =~ /rpffcstgroup/) { + if($extract eq 0) { + $table_query = "SELECT distinct $select_string from rpffcstgroup join rpffcstpoint rp on rp.group_id = rpffcstgroup.group_id join location l on l.lid = rp.lid where l.hsa = '$wfoID';"; + } else { + my $rpgroup_extract_detail = $extract_detail; + $rpgroup_extract_detail =~ s/lid/l.lid/g; + $table_query = "SELECT distinct $select_string from rpffcstgroup join rpffcstpoint rp on rp.group_id = rpffcstgroup.group_id join location l on l.lid = rp.lid where l.hsa is not null $rpgroup_extract_detail;"; + } + } elsif($table_name =~ /vtecevent/) { + if($extract eq 0) { + $table_query = "SELECT $select_string FROM vtecevent WHERE vtecevent.geoid in (select location.lid from location where location.hsa = '$wfoID') ;"; + } else { + my $vtec_extract_detail = $extract_detail; + $vtec_extract_detail =~ s/lid/geoid/g; + print "vtec_extract_detail: $vtec_extract_detail\n"; + $table_query = "SELECT $select_string FROM vtecevent WHERE vtecevent.geoid in (select location.lid from location where location.hsa is not null) $vtec_extract_detail;"; + } + } elsif($table_name eq "height" || $table_name =~ /temperature/ || $table_name =~ /curpp/ || $table_name =~ /curpc/ || $table_name eq "discharge"){ + my $cutoff_dtime = getcutoffdate(); + if($extract eq 0) { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid AND location.hsa = '$wfoID') and obstime > '$cutoff_dtime' ORDER BY lid;"; + } else { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid ) and obstime > '$cutoff_dtime' $extract_detail ORDER BY lid;"; + } + } elsif($table_name =~ /fcstheight/ || $table_name =~ /fcstdischarge/) { + my $cutoff_dtime = getcutoffdate(); + if($extract eq 0) { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid AND location.hsa = '$wfoID') and basistime > '$cutoff_dtime' ORDER BY lid;"; + } else { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid) and basistime > '$cutoff_dtime' $extract_detail ORDER BY lid;"; + } + } elsif($lid_flag == 1) { + if($extract eq 0) { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid AND location.hsa = '$wfoID') $extract_detail ORDER BY lid;"; + } else { + $table_query = "SELECT $select_string FROM $table_name WHERE exists (SELECT lid FROM location WHERE location.lid = $table_name.lid) $extract_detail ORDER BY lid;"; + } + } else { + $table_query = "SELECT $select_string FROM $table_name\;"; + } + } + } elsif($type eq "RFC") { + if($table_name =~ /location/) { + $table_query = "SELECT $select_string FROM location WHERE location.rfc='$rfcID' $extract_detail ORDER BY lid;"; + } elsif($lid_flag == 1) { + $table_query = "SELECT $select_string from $table_name where exists (select lid from location where +location.lid = $table_name.lid and location.rfc='$rfcID') $extract_detail ORDER BY lid;"; + # $table_query = "SELECT $select_string from $table_name where exists (select lid from location where +#location.lid=rating.lid and location.rfc='$rfcID') $extract_detail ORDER BY lid;"; + } else { + $table_query = "SELECT $select_string FROM $table_name\;"; + } + } + + # print the query for log purpose and execute the query + print "$table_query\n\n" if($verbose); + warn "$table_query\n\n"; + $record_count = 0; + eval + { + $st = $db->prepare($table_query); + $row = $db->selectall_arrayref($st,{Slice => {}}); + #$st->execute() or die "Cannot execute: ".$st->errstr(); + }; + if ($@) + { + print "$@\n" if($verbose); + warn "$@\n"; + $xml_string .= " \n"; + $query_error_flag = 1; + } + + # if no db error continue adding info to xml file for the table. + if($query_error_flag == 0) + { + $numrows = $st->rows; + print "Number of records obtained: $numrows\n" if($verbose); + warn "Number of records obtained: $numrows\n"; + if ($numrows == 0) + { + $xml_string .= "
\n"; + } + else + { + $xml_string .= "
\n"; + } + + foreach my $sref (@$row) + { + %infohash=%{$sref}; + #print record number to xml file + $xml_string .= " \n \n"; + + #print primary key to xml file + my $pk_count = 0; + foreach my $pk (@pk_output) + { + if($pk =~ /$table_name\.(.*)/) + { + $pk_name=$1; + #$infohash{$pk_name}=~ s/\r|\n//g; + $xml_string .= " <$pk>$infohash{$pk_name}\n"; + $pk_count++; + } + } + $xml_string .= " \n \n"; + @select_array = split(/,/, $field_string); + #start printing fields to xml file + my $field_count = 0; + foreach my $select (@select_array) + { + if($select =~ /.*$table_name\.(.*)/) + { + $field_name = $1; + if($infohash{$field_name} !~/^\s*$/) + { + #$infohash{$field_name} =~ s/\r|\n//g; + $xml_string .= " <$select>$infohash{$field_name}\n"; + } + else + { + $xml_string .= " <$select/>\n"; + } + $field_count++; + } + } + $xml_string .=" \n"; + $xml_string .=" \n"; + $record_count++; + } + + } + if($numrows != 0 && $query_error_flag == 0) + { + $xml_string .="
\n"; + } + @select_array = (); + $field_string = ""; + + print "\n---------------\n" if($verbose); + warn "\n---------------\n"; + +} +$xml_string .="\n"; + +if ($type eq "WFO" && $wfoID eq 0) +{ + my $hsa_admin_query = "SELECT admin.hsa FROM admin;"; + my $st_admin; + eval + { + $st_admin = $db->prepare($hsa_admin_query); + $st_admin->execute() or die "Cannot execute: ".$st_admin->errstr(); + }; + if ($@) + { + print "$@\n" if($verbose); + warn "$@\n"; + } + while ( defined ( my $row = $st_admin->fetchrow_arrayref() ) ) + { + $wfoID = @$row[0]; + } + +} + +if($type eq "WFO") +{ + $file_name = "$wfoID\_from-$office\_nrldb.xml"; +} +elsif($type eq "RFC") +{ + $file_name = "$rfcID\_from-$office\_nrldb.xml"; +} + + +#determine output file +if($outFile eq 0) +{ + $outFile = $file_name; +} + +my $outDir; + +if( -e "/awips/hydroapps/public/bin/get_apps_defaults"){ + $outDir = `/awips/hydroapps/public/bin/get_apps_defaults.LX nrldb_data`; + + chomp($outDir); +} else { + print "Could not access /awips/hydroapps/public/bin/get_apps_defaults.LX. Exiting"; + exit -1; +} + +$outFile = $outDir . "/" . $outFile; +open(XMLFILE, ">$outFile") || die "Could not open $outFile for writing.\n$!\nExiting\n"; +printf XMLFILE "$xml_string"; +close(XMLFILE); + +my $end = $db->disconnect; +zip_xml($outFile); +} + +sub zip_xml +{ +my $filename = shift; +my $zip_string; + + $zip_string = "zip -j $filename.zip $filename"; + print "$zip_string\n" if($verbose); + warn "$zip_string\n"; + my $zip_exe = `$zip_string`; + print "$zip_exe\n" if($verbose); + warn "$zip_exe\n"; + print "Failed: \"$zip_string\"\n" if ($? && $verbose); + warn "Failed: \"$zip_string\"\n" if $?; +} + + +sub read_control_file +{ +my @fields_all; +my @tables; +my @fields; +my $table_name; +my $control_file; + +if($localControlFile eq 0) +{ + if($type eq "WFO") + { + $control_file = "${conf_dir}/nrldb_control_wfo"; + } + elsif($type eq "RFC") + { + $control_file = "${conf_dir}/nrldb_control_rfc"; + } +} +else +{ + $control_file = $localControlFile; +} +open(FILE, "$control_file") || die "Could not open control file: $control_file\n$!\nExiting\n"; +my @infile = ; +close(FILE); + +foreach my $line (@infile) +{ +chomp($line); + if($line =~ /^#.*$/) + { + next; + } + elsif($line =~ /\[(.*)\]/) + { + $table_name = $1; + push (@tables, $table_name); + } + elsif($line =~ /^(fields)/) + { + $line =~ /fields = (.*)/; + @fields = split(/,/, $1); + + foreach my $tmp_field (@fields) + { + $tmp_field =~ s/\s*//; + push(@fields_all, "$table_name.$tmp_field"); + } + } +} + + +return (\@tables, \@fields_all); +} + +sub extract_detail() +{ + +my $wfo = $office; +my $wfo_fh_pointer = 0; +my $info_found = 0; +my ($ex_type, $ex_list); +my @extract_lid; +my $uclid; +my $compare_symbol; +my $extract_query = ''; + +open(FILE, "nrldb_extract") || die "Could not open detail extract file nrldb_extract:\n$!\nExiting\n"; +my @infile = ; +close(FILE); + + foreach my $line (@infile) + { + chomp($line); + if($line =~ m/type:\s*(\w*)/) + {$ex_type= $1;} + if($line =~ m/list:\s*(.*)/) + { + $ex_list= $1; + if(defined($ex_type) && defined($ex_list)) + {$info_found = 1;} + } + + if($info_found eq 1) + {last;} + } + if($info_found eq 1) + { + print "EXTRACT: $ex_type, [$ex_list]\n" if($verbose); + warn "EXTRACT: $ex_type, [$ex_list]\n"; + @extract_lid = split(/,/,$ex_list); + + if(lc($ex_type) eq 'only') + {$compare_symbol = '=';} + elsif(lc($ex_type) eq 'except') + {$compare_symbol = '!=';} + else + { + print "Undefined extraction type '$ex_type', should be only|except\n" if($verbose); + warn "Undefined extraction type '$ex_type', should be only|except\n"; + return($extract_query); + } + # The following has been modified by Mark Armstrong HSD + # Originally, the query for multiple lids using the "only" extract + # was incorrect. It used the AND condition for each lid which + # would never be true. I added another if condition and a new + # for loop to handle this case. + if(lc($ex_type) eq 'only'){ + my $count = 0; + $extract_query=" AND ("; + foreach my $lid (@extract_lid) + { + if($lid eq '') + {next;} + + $uclid=uc($lid); + $uclid =~ s/\s*//g; + if ( $count eq 0) + { + $extract_query .= " lid $compare_symbol '$uclid'"; + } + else + { + $extract_query .= " OR lid $compare_symbol '$uclid'"; + } + $count = $count + 1; + } + $extract_query .= ") "; + } + else{ + foreach my $lid (@extract_lid) + { + if($lid eq '') + {next;} + + $uclid=uc($lid); + $uclid =~ s/\s*//g; + $extract_query .= " AND lid $compare_symbol '$uclid'"; + + } + } + } + return($extract_query); +} + +sub read_config_file() +{ + +my $dbname; +my $host; +my $pass; +my $user; +my $nrldb_host; +my $site_conf; +my $backup_host; +my $conf_file; + +if( -e "/awips/hydroapps/public/bin/get_apps_defaults") +{ + $conf_dir = `/awips/hydroapps/public/bin/get_apps_defaults.LX nrldb_config`; + chomp($conf_dir); + $conf_file = "${conf_dir}/nrldb.conf"; +} +else +{ + print "nrldb_conf token not specified. Exiting"; + exit -1; +} +open(FILE, "${conf_file}") || die "Could not open configuration ${conf_file}:\n$!\nExiting\n"; +my @infile = ; +close(FILE); + + foreach my $line (@infile) + { + chomp($line); + if($line =~ /(^\s*dbname\s*=\s*"(.*)")/) + { + $dbname = "$2"; + } + elsif($line =~ /(^\s*dbhost\s*=\s*"(.*)")/) + { + $host = "$2"; + } + elsif($line =~ /(^\s*dbpass\s*=\s*"(.*)")/) + { + $pass = "$2"; + } + elsif($line =~ /(^\s*dbuser\s*=\s*"(.*)")/) + { + $user = "$2"; + } + elsif($line =~ /(^\s*nrldb_host\s*=\s*"(.*)")/) + { + $nrldb_host = "$2"; + } + elsif($line =~ /(^\s*site\s*=\s*"(.*)")/) + { + $site_conf = "$2"; + } + elsif($line =~ /(^\s*backup_host\s*=\s*"(.*)")/) + { + $backup_host = "$2"; + } + + } + return($dbname, $host, $user, $pass, $nrldb_host, $site_conf, $backup_host); +} + + +sub xml_parse +{ +my $xmlfile = $inFile; # the file to parse +my $lineCount = 0; +my @rawLine; +my $last_f; +my $record_num; +my $table; +my ($i, $j, $k); +my ($PK_name, $PK_value, $Field_name, $Field_value); +sub insertValues($table, $record_num, $PK_name, $PK_value, $Field_name, $Field_value); + +print "Parsing and Inserting Values from $xmlfile into database\n\n" if($verbose); +warn "Parsing and Inserting Values from $xmlfile into database\n\n"; + +open(XML_FH, "$xmlfile") or die("Cant open file $xmlfile for reading: $!\nExiting\n"); +while () +{ + # $_ is the line that has set. + $rawLine[$lineCount] = "$_"; + $lineCount++; +} + + + +close(XML_FH); + +$i=0; + + while (!$last_f) + { + if ($rawLine[$i] =~ m//) + { + print "Current Table: $1\n" if($verbose); + warn "Current Table: $1\n"; + $table = $1; + while($rawLine[$i] !~ m/<\/Table>/) + { + if($rawLine[$i] =~ //) + { + $record_num = $1; + while ($rawLine[$i] !~ m/<\/Record>/) + { + if($rawLine[$i] =~ //) + { $i++; + $j = 0; + while($rawLine[$i] !~ m/<\/PK>/) + { + if($rawLine[$i] =~ m/<$table\.(.*?)>(.*)<\/$table\..*>/) + { + $$PK_name[$j] = $1; + $$PK_value[$j] = $2; + $j++; + } + elsif($rawLine[$i] =~ m/<$table\.(.*)\/>/) + { + $$PK_name[$j] = $1; + $$PK_value[$j] = "NULL"; + $j++; + } + elsif($rawLine[$i] =~ m/<$table\.(.*?)>.*/) + { + + {$$PK_name[$k] = $1;} + $$PK_value[$j] = ''; + do + { + $$PK_value[$j] .= $rawLine[$i]; + $i++; + } until ($rawLine[$i] =~ m/<\/$table\..*>$/); + $$PK_value[$j] .= $rawLine[$i]; + $$PK_value[$j] =~ s/^\s*<$table\.(.*)>//g; + $$PK_value[$j] =~ s/<\/$table\..*>$//g; #/ + $j++; + } + $i++; + } + } + if($rawLine[$i] =~ //) + { $i++; + $k = 0; + while($rawLine[$i] !~ m/<\/Fields>/) + { + if($rawLine[$i] =~ m/<$table\.(.*?)>(.*)<\/$table\..*>/) + { + $$Field_name[$k] = $1; + $$Field_value[$k] = $2; + $k++; + } + elsif($rawLine[$i] =~ m/<$table\.(.*)\/>/) + { + $$Field_name[$k] = $1; + $$Field_value[$k] = "NULL"; + $k++; + } + elsif($rawLine[$i] =~ m/<$table\.(.*?)>.*/) + { + + {$$Field_name[$k] = $1;} + $$Field_value[$k] = ''; + do + { + $$Field_value[$k] .= $rawLine[$i]; + $i++; + } until ($rawLine[$i] =~ m/<\/$table\..*>$/); + $$Field_value[$k] .= $rawLine[$i]; + $$Field_value[$k] =~ s/^\s*<$table\.(.*)>//g; + $$Field_value[$k] =~ s/<\/$table\..*>$//g; #/ + $k++; + } + $i++; + } + } + $i++; + } + &insertValues($table, $record_num, $PK_name, $PK_value, $Field_name, $Field_value); + $#$PK_name = -1; $#$PK_value = -1; $#$Field_name = -1; $#$Field_value = -1; + $total_count++; + } + $i++; + } + print "\tTotal Inserts: $insert_count\n" if($verbose); + warn "\tTotal Inserts: $insert_count\n"; + print "\tTotal Updates: $update_count\n" if($verbose); + warn "\tTotal Updates: $update_count\n"; + print "\tTotal Errors: $error_count\n" if($verbose); + warn "\tTotal Errors: $error_count\n"; + print "\tTOTAL: $total_count\n\n" if($verbose); + warn "\tTOTAL: $total_count\n\n"; + $insert_count = 0; + $update_count = 0; + $error_count = 0; + $total_count = 0; + } + elsif ($rawLine[$i] =~ /<\/NRLDB>/) + {$last_f = 1;} + else + {$i++;} + } + +} + +sub get_delete_list +{ + my @list; + my $table; + + open(FILE, "${conf_dir}/nrldb_control_delete") || die "Could not open detail extract file ${conf_dir}/nrldb_control_delete:\n$!\nExiting\n"; + my @infile = ; + close(FILE); + + foreach my $line (@infile) + { + chomp($line); + if($line =~ m/^\s*#/) + {next;} + + if($line =~ m/^\s*\w+\s*$/) + { + $line =~ s/\s*//g; + $table=lc($line); + push(@list, $table); + } + } + + return(\@list); +} + +sub deleteValues +{ + my $deleteTable = shift; + my $deleteWFO = $office; + my $lid_flag = lid_check($deleteTable); + my ($delete_query, $st); + + my ($delete_detail, $total); + + if($lid_flag == 1) + { + ($delete_detail, $total)=getDeleteLid($deleteTable); + if($total !=0) + { + $delete_query = "DELETE FROM $deleteTable $delete_detail\;"; + print "DELETE: $delete_query\n"; + } + } + else + { + $delete_query = "DELETE FROM $deleteTable\;"; + } + + eval + { + $st = $db->prepare($delete_query); + $st->execute() or die "Cannot execute: ".$st->errstr(); + }; + if($@) + {print "$@\n" if($verbose); warn "$@\n";} + +} + + +sub getDeleteLid +{ + +my $xmlfile = $inFile; # the file to parse +my $lineCount = 0; +my @rawLine; +my $last_f; +my $record_num; +my $table; +my ($i, $j, $k); +my $lid_name; + +my $deleteTable = shift; +my $total_count = 0; + +open(XML_FH, "$xmlfile") or die("Cant open file $xmlfile for reading: $!\nExiting\n"); +while () +{ + # $_ is the line that has set. + $rawLine[$lineCount] = "$_"; + $lineCount++; +} + +close(XML_FH); + +$i=0; +my $delete_str = ""; +my $last_lid = -1; + while (!$last_f) + { + if ($rawLine[$i] =~ m/
/) + { + print "Delete Table: $1\n" if($verbose); + warn "Delete Table: $1\n"; + $table = $1; + while($rawLine[$i] !~ m/<\/Table>/) + { + if($rawLine[$i] =~ //) + { + $record_num = $1; + while ($rawLine[$i] !~ m/<\/Record>/) + { + if($rawLine[$i] =~ //) + { $i++; + while($rawLine[$i] !~ m/<\/PK>/) + { + if($rawLine[$i] =~ m/<$table\.lid>(.*)<\/$table\.lid>/) + { + if(($last_lid != -1) && ($last_lid eq $1)) + {$i++; next;} + #print "$1\n"; + if ($total_count == 0) + { + $delete_str .= "WHERE $table.lid = '$1'"; + } + else + { + $delete_str .= " OR $table.lid = '$1'"; + } + + $last_lid = $1; + + } + $i++; + } + } + $i++; + } + $total_count++; + } + $i++; + } + print "\tTotal Delete LIDs: $total_count\n" if($verbose); + warn "\tTotal Delete LIDs: $total_count\n"; + $last_f = 1; + } + elsif ($rawLine[$i] =~ /<\/NRLDB>/) + {$last_f = 1;} + else + {$i++;} + } + #print "$delete_str, $total_count\n"; + return ($delete_str, $total_count); + +} + + +sub insertValues($table, $record_num, $PK_name, $PK_value, $Field_name, $Field_value) +{ + my $num; + my ($fields, $values); + my ($update_set, $update_where); + my $Field_value_quoted; + my $table = shift; + my $record_num = shift; + my $PK_name = shift; + my $PK_value = shift; + my $Field_name = shift; + my $Field_value = shift; + my $update_flag = 0; + my $st_handle; + my $insertrows; + + for($num = 0; $num <= $#$Field_value; $num++) + { + if($num == 0) + { + $fields = "($$Field_name[$num]"; + if($$Field_value[$num] ne "NULL") + { + $$Field_value[$num] = $db->quote($$Field_value[$num]); + $values = "($$Field_value[$num]"; + $update_set = "$$Field_name[$num]=$$Field_value[$num]"; + } + else + { + $values = "($$Field_value[$num]"; + $update_set = "$$Field_name[$num]=$$Field_value[$num]"; + } + } + else + { + $fields .= ", $$Field_name[$num]"; + if($$Field_value[$num] ne "NULL") + { + $$Field_value[$num] =~ s/\n//g; + $$Field_value[$num] =~ s/\r//g; + $$Field_value[$num] = $db->quote($$Field_value[$num]); + $values .= ", $$Field_value[$num]"; + $update_set .= ", $$Field_name[$num]=$$Field_value[$num]"; + } + else + { + $values .= ", $$Field_value[$num]"; + $update_set .= ", $$Field_name[$num]=$$Field_value[$num]"; + } + } + } + for($num = 0; $num <= $#$PK_name; $num++) + { + if($num == 0) + { + $$PK_value[$num] = $db->quote($$PK_value[$num]); + $update_where = "$$PK_name[$num]=$$PK_value[$num] "; + } + else + { + $$PK_value[$num] = $db->quote($$PK_value[$num]); + $update_where .= "AND $$PK_name[$num]=$$PK_value[$num]"; + } + } + + $fields .= ")"; + $values .= ")"; + my $insert_cmd = "INSERT INTO $table $fields VALUES $values\;"; + my $update_cmd = "UPDATE $table SET $update_set WHERE $update_where\;"; + + eval { + $insert_count++; + $st_handle = $db->prepare($insert_cmd); + $st_handle->execute() or die "Cannot execute: ".$st_handle->errstr(); + $insertrows = $st_handle->rows(); + if($insertrows == 0) + { + $insert_count--; + $error_count++; + print "ZERO ROWS FOR QUERY: $insert_cmd\n\n" if($verbose); + warn "ZERO ROWS FOR QUERY: $insert_cmd\n\n"; + } + }; + + if ($@) { + if($@ =~ /duplicate key/) + { + $update_flag = 1; + $insert_count--; + } + else + { + print "$@\n" if($verbose); + warn "$@\n"; + $insert_count--; + $error_count++; + print "INSERT ERROR ON QUERY: $insert_cmd\n\n" if($verbose); + warn "INSERT ERROR ON QUERY: $insert_cmd\n\n"; + + } + } + + if($update_flag == 1) + { + eval { + $update_count++; + $st_handle = $db->prepare($update_cmd); + $st_handle->execute() or die "Cannot execute: ".$st_handle->errstr(); + $insertrows = $st_handle->rows(); + if($insertrows == 0) + { + $update_count--; + $error_count++; + print "ZERO ROWS FOR QUERY: $update_cmd\n\n" if($verbose); + warn "ZERO ROWS FOR QUERY: $update_cmd\n\n"; + } + }; + + if ($@) { + print "$@\n" if($verbose); + warn "$@\n"; + $update_count--; + $error_count++; + print "UPDATE ERROR ON QUERY: $update_cmd\n\n" if($verbose); + warn "UPDATE ERROR ON QUERY: $update_cmd\n\n"; + } + } + +} + + +sub db_connect +{ +my $dbname = shift; +my $host = shift; +my $user = shift; +my $pass = shift; + +my %db_attr = ( + PrintError => 0, + RaiseError => 0, +); + +my $dsn = "DBI:Pg:dbname=$dbname;host=$host"; +my $db = DBI->connect($dsn, $user, $pass, \%db_attr) or die "Can't connect() to database $dbname: $DBI::errstr"; +return ($db); +} + +sub upload_xml +{ + print "---UPLOAD XML FILE----\n" if($verbose); + warn "---UPLOAD XML FILE----\n"; + my $upload_string = "rsync -av --chmod=ugo+rw $outFile.zip $nrldb_host\::nrldb_xml/"; + print "$upload_string\n" if($verbose); + warn "$upload_string\n"; + my $upload_exe = `$upload_string`; + print "$upload_exe\n" if($verbose); + warn "$upload_exe\n"; + print "Failed: \"$upload_string\"\n" if ($? && $verbose); + warn "Failed: \"$upload_string\"\n" if $?; + return; +} +sub download_control_file +{ + my $office_type = shift; + my $download_string; + print "---DOWNLOAD $office_type CONTROL FILE----\n" if($verbose); + warn "---DOWNLOAD $office_type CONTROL FILE----\n"; + + if ($office_type eq "WFO") + { + $download_string = "rsync -av $nrldb_host\::nrldb_control/nrldb_control_wfo ${conf_dir}/"; + } + elsif ($office_type eq "RFC") + { + $download_string = "rsync -av $nrldb_host\::nrldb_control/nrldb_control_rfc ${conf_dir}/"; + } + print "$download_string\n" if($verbose); + warn "$download_string\n"; + my $download_exe = `$download_string`; + print "$download_exe\n" if($verbose); + warn "$download_exe\n"; + print "Failed: \"$download_string\"\n" if ($? && $verbose); + warn "Failed: \"$download_string\"\n" if $?; + return; +} + +sub getdate() +{ +my ($Second, $Minute, $Hour, $Day, $Month, $Year, $WeekDay, $DayOfYear, $IsDST) = localtime(time) ; +my $RealMonth = $Month + 1 ; # Months of the year are not zero-based +my $FixedYear; + +if ($Hour < 10) +{ + $Hour = "0" . $Hour +} + +if ($Minute < 10) +{ + $Minute = "0" . $Minute +} + +if ($Second < 10) +{ + $Second = "0" . $Second +} + +if ($RealMonth < 10) +{ + $RealMonth = "0" . $RealMonth; +} + +if ($Day < 10) +{ + $Day = "0" . $Day; +} + +if ($Year >= 100) +{ + $FixedYear = $Year - 100; +} +else +{ + $FixedYear = $Year; +} + +if ($FixedYear < 10) +{ + $FixedYear = "0" . $FixedYear; +} + +my $clean_date = "$Hour:$Minute:$Second $RealMonth/$Day/$FixedYear"; + +return($clean_date); +} + +sub lid_check { + my $table_name = shift; + my $at; + my $lid_flag = 0; + + my $query_column1 = "SELECT c.oid + FROM pg_catalog.pg_class c + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE pg_catalog.pg_table_is_visible(c.oid) + AND c.relname ~ '^$table_name\$'"; + + my $attribute_query = "SELECT a.attname + FROM pg_catalog.pg_attribute a + WHERE a.attnum > 0 AND NOT a.attisdropped + AND a.attrelid = ($query_column1) + ORDER BY a.attnum;"; + + eval { + $at = $db->prepare($attribute_query); + $at->execute() or die "Cannot execute: ".$at->errstr(); + }; + if($@) { + print "$@\n"; + } + + while ( defined ( my $attribues = $at->fetchrow_arrayref() ) ) { + if(@$attribues[0] =~ /^lid$/) { + $lid_flag = 1; + } + } + +return ($lid_flag); +} + +BEGIN { + use CGI::Carp qw(carpout); + my $logDir; + if( -e "/awips/hydroapps/public/bin/get_apps_defaults"){ + $logDir = `/awips/hydroapps/public/bin/get_apps_defaults.LX nrldb_log`; + chomp($logDir); + } else { + print "Could not access /awips/hydroapps/public/bin/get_apps_defaults.LX. Exiting\n"; + exit -1; + } + print "log dirlogDir\n"; + my $log = "${logDir}/nrldb.log"; + open(LOG, ">>$log") or die "Unable to open $log. $! "; + carpout(*LOG); +} + +END { + my $date = `date`; + print LOG "End $0 at $date\tElapsed time: " . (time - $^T) . " seconds\n\n"; + close LOG; +} + +sub getcutoffdate() +{ +my ($Second, $Minute, $Hour, $Day, $Month, $Year, $WeekDay, $DayOfYear, $IsDST) = gmtime(time-172800) ; +my $RealMonth = $Month + 1 ; # Months of the year are not zero-based +my $FixedYear; + +if ($Hour < 10) +{ + $Hour = "0" . $Hour +} + +if ($Minute < 10) +{ + $Minute = "0" . $Minute +} + +if ($Second < 10) +{ + $Second = "0" . $Second +} + +if ($RealMonth < 10) +{ + $RealMonth = "0" . $RealMonth; +} + +if ($Day < 10) +{ + $Day = "0" . $Day; +} + + $FixedYear = $Year + 1900; + +my $clean_date = "$FixedYear-$RealMonth-$Day $Hour:$Minute"; + +return($clean_date); +} diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/run_update_nrldb.ksh b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/run_update_nrldb.ksh new file mode 100755 index 0000000000..d35a1ce16e --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/run_update_nrldb.ksh @@ -0,0 +1,21 @@ +#!/bin/ksh + +RUN_FROM_DIR=`dirname $0` +# set up SOME environment variables for WHFS applications +export PGSQL_DRIVER_DIR=/awips2/cave/plugins/org.postgres_9.2.0 +. $RUN_FROM_DIR/../../set_hydro_env +. $RUN_FROM_DIR/../../check_app_context +export APPS_DEFAULTS=~/caveData/common/base/hydro/Apps_defaults +export APPS_DEFAULTS_SITE=~/caveData/common/site/${AW_SITE_IDENTIFIER}/hydro/Apps_defaults +export PGUSER="awips" +export BIN_DIR=`get_apps_defaults whfs_bin_dir"` +export NRLDB_LOG=`get_apps_defaults nrldb_log` +export NRLDB_CONFIG=`get_apps_defaults nrldb_config` +export NRLDB_DATA=`get_apps_defaults nrldb_data` +#export NRLDB_TMP=`get_apps_defaults nrldb_tmp` +export db_name=`get_apps_defaults db_name` +export PGHOST=`get_apps_defaults pghost` + +$RUN_FROM_DIR/update_nrldb.pl + +exit 0 diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/send_nrldb_update.sh b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/send_nrldb_update.sh new file mode 100755 index 0000000000..946a1464ea --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/send_nrldb_update.sh @@ -0,0 +1,167 @@ +#!/bin/sh +############################################################################### +# This script is run at the field office to send ad-hoc updates to the NRLDB +# server, then on to the AHPS CMS. It can be run at any time. It is designed +# to send small, time-sensitive updates to the CMS. It takes two argument +# lists:-table table names (comma-separated) and -lid lid names +# (comma-separated). It parses the arguments, selects the updated data from +# the database and builds an SQL formatted text file for use on the nrldb and +# CMS databases. The SQL file contains a delete staement that deletes the +# pre-existing data for the lid/table combinations, before running the inserts +# +# Usage: send_nrldb_update.sh -table ,,... -lid ,,... +# Example: send_nrldb_update.sh -table rating,floodstmt -lid BRKM2,CBEM2 +# +if [ $# -ne 4 ] +then + echo "Incorrect number of arguments entered: $#" + echo "Correct Arguments are:" + echo "send_nrldb_update.sh -table table1,table2 -lid lid1,lid2" + echo "Any number of tables and lids may be specified, but they need to be in a comma separated list with no spaces between commas and table/lid names" + exit 0 +fi +# set up SOME environment variables for NRLDB applications + +# get the nrldb host and wfo from the nrldb.conf file/database +nrldb_host=`grep nrldb_host $NRLDB_CONFIG/nrldb.conf | cut -d= -f2 | sed 's/"//g' | sed 's/ //g'` +echo "DB NAME: $db_name" +wfo=`psql -h $PGHOST -d $db_name -c "select hsa from admin;" | tail -3 | head -1 | sed -e 's/ //g'` +echo `date` +echo "WFO $wfo" + +# create the final SQL file that will be sent to the NRLDB host +timestamp=`date +%Y%m%d%H%N` +sql_file="${wfo}_update_${timestamp}.sql" +if [ -f $sql_file ] +then + rm $sql_file +fi + +# build the list of tables/lids to send +lid_list="XXXXX" +table_list="XXXXX" +while [ $# -gt 0 ] +do + case "$1" in + -lid) lid_list="$2,";shift;; + -table) table_list="$2,";shift;; + *) break;; + esac + shift +done + +# set the last update information for update_nrldb.pl to use +echo `date` > ${NRLDB_LOG}/last_nrldb_update.txt +up_lid_list=`echo $lid_list | sed 'y/abcdefghijklmnopqrstuvwxyz/ABCDEFGHIJKLMNOPQRSTUVWXYZ/'` +echo "lid list: $up_lid_list" >> ${NRLDB_LOG}/last_nrldb_update.txt +echo "table_list: $table_list" >> ${NRLDB_LOG}/last_nrldb_update.txt + +#loop through the tables/lids +if [ $table_list != "XXXXX" ] +then + pos=1 + table="XXXXX" + ltable=`echo $table | wc -m` + while [ $ltable -gt 4 ] + do + table=`echo $table_list | cut -d"," -f$pos` + pos=`expr $pos + 1` + ltable=`echo $table | wc -m` + if [ $ltable -gt 4 ] + then + lid="XXXXX" + lpos=1 + llid=`echo $lid | wc -m` + while [ $llid -gt 3 ] + do + lid=`echo $up_lid_list | cut -d"," -f$lpos` + lpos=`expr $lpos + 1` + llid=`echo $lid | wc -m` + if [ $llid -gt 3 ] + then + # fetch the values from the DB and edit them + export PGUSER=awips + touch $NRLDB_TMP/update.txt + chmod ugo+rw $NRLDB_TMP/update.txt + ls -l $NRLDB_TMP/update.txt + psql -h $PGHOST -d $db_name -c "copy (select * from $table where lid = '$lid') to '$NRLDB_TMP/update.txt' with delimiter '|';" + cp $NRLDB_TMP/update.txt ${NRLDB_DATA}/update.txt + sed -f ${NRLDB_CONFIG}/sed_script.txt ${NRLDB_TMP}/update.txt > ${NRLDB_DATA}/update11.txt + sed -e "s/|/'|'/g" ${NRLDB_DATA}/update11.txt > ${NRLDB_DATA}/update1.txt + sed -e "s/^/insert into $table values('/g" ${NRLDB_DATA}/update1.txt > ${NRLDB_DATA}/update2.txt + sed -e "s/$/');/g" ${NRLDB_DATA}/update2.txt > ${NRLDB_DATA}/update3.txt + sed -e "s/|/,/g" ${NRLDB_DATA}/update3.txt > ${NRLDB_DATA}/update4.txt + if [ -f "${NRLDB_DATA}/update.txt" ] + then + update_lines=`wc -l "${NRLDB_DATA}/update.txt" | cut -d" " -f1` + else + echo "No update file found". + update_lines=0 + fi + if [ $update_lines -gt 0 ] + then + if [ $table != "location" -a $table != "riverstat" ] + then + echo "delete from $table where lid = '$lid';" >> ${NRLDB_DATA}/$sql_file + fi + cat ${NRLDB_DATA}/update4.txt >> ${NRLDB_DATA}/$sql_file + fi + # location and riverstat require a special forecast since they have dependent tables via foreign keys + if [ $table = "location" ] + then + sql_stmt="update location set lid = '$lid'" + for col in county coe cpm detail elev hdatum hsa hu lat lon lremark lrevise name network rb rfc sbd sn state waro wfo wsfo type des det post stntype tzone + do + psql -h $PGHOST -d $db_name -c "select $col from location where lid = '$lid' and $col is not null;" > ${NRLDB_DATA}/update.txt + ct_zero=`grep -c "0 row" ${NRLDB_DATA}/update.txt` + if [ $ct_zero -eq 0 ] + then + export val=`cat ${NRLDB_DATA}/update.txt | head -3 | tail -1 | cut -c2-80` + new_val=`echo "$val" | sed -f ${NRLDB_CONFIG}/sed_script.txt` + sql_stmt="$sql_stmt, $col = '$new_val'" + fi + done + sql_stmt="$sql_stmt where lid = '$lid';" + echo $sql_stmt >> ${NRLDB_DATA}/$sql_file + + elif [ $table = "riverstat" ] + then + sql_stmt="update riverstat set lid = '$lid'" + for col in primary_pe bf cb da response_time threshold_runoff fq fs gsno level mile pool por rated lat lon remark rrevise rsource stream tide backwater vdatum action_flow wstg zd ratedat usgs_ratenum uhgdur use_latest_fcst + do + psql -h $PGHOST -d $db_name -c "select $col from riverstat where lid = '$lid' and $col is not null;" > ${NRLDB_DATA}/update.txt + ct_zero=`grep -c "0 row" ${NRLDB_DATA}/update.txt` + if [ $ct_zero -eq 0 ] + then + export val=`cat ${NRLDB_DATA}/update.txt | head -3 | tail -1 | cut -c2-80` + new_val=`echo "$val" | sed -f ${NRLDB_CONFIG}/sed_script.txt` + sql_stmt="$sql_stmt, $col = '$new_val'" + fi + done + sql_stmt="$sql_stmt where lid = '$lid';" + echo $sql_stmt >> ${NRLDB_DATA}/$sql_file + fi + fi + done + fi + + done + + # send the SQL file to the NRLDB server + if [ -f ${NRLDB_DATA}/$sql_file ] + then + rsync -av ${NRLDB_DATA}/$sql_file ${nrldb_host}\::nrldb_update/ + echo "SQL file: $sql_file created for lids: $up_lid_list and tables: $table_list" + else + echo "No SQL file created. Database contained no entries for lids: $up_lid_list and tables: $table_list" + fi +fi + +# remove the temp files to keep the directory clean +for temp_file in ${NRLDB_DATA}/update.txt ${NRLDB_DATA}/update11.txt ${NRLDB_DATA}/update1.txt ${NRLDB_DATA}/update2.txt ${NRLDB_DATA}/update3.txt ${NRLDB_DATA}/update4.txt +do + if [ -f $temp_file ] + then + rm $temp_file + fi +done diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/update_nrldb.pl b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/update_nrldb.pl new file mode 100755 index 0000000000..97669495f3 --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/bin/update_nrldb.pl @@ -0,0 +1,248 @@ +#!/usr/bin/perl +################################################################################ +# update_nrldb.pl is the GUI for the Ad-Hoc update process. ## This process was put in place so that WFOs could update information # +# between daily runs of the NRLDB update process. The information is # +# collected at the WFO, sent to the NRLDB central server and then forwarded to # +# CMS servers outside of the AWIPS firewall. # +# # +# Developer: Mark Armstrong (OCWWS/HSD) # +# Developed 2011 - Modified for AWIPS2 2013 # +################################################################################ + +use Tk; +use strict; +use warnings; +use AppConfig qw(:expand :argcount); +use DBI; + +our $BIN_DIR = `echo \$BIN_DIR`; +chomp($BIN_DIR); +our $NRLDB_LOG = `echo \$NRLDB_LOG`; +chomp($NRLDB_LOG); + +my $lids; +my $tables; + +# Set up some inial configuration. Most of this comes from the hydroGen input file: hg.cfg +$ENV{HYDROGENHOME} = "/awips/hydroapps/HydroGen" if ! defined $ENV{HYDROGENHOME}; +my %cfg = ( DEBUG => 0, # debug mode on or off + PEDANTIC => 0, # be patient with warnings/errors + CREATE => 1, # create variables, defining not required... + GLOBAL => { # for all config options unless overridden... + EXPAND => EXPAND_ALL, # expand ~, $ENV{*}, and $(var) + ARGCOUNT => ARGCOUNT_ONE, # each config expects an arg unless overriden... + ARGS => '=s' # each arg is a string unless overriden + } + ); + +my $config = AppConfig->new(\%cfg); # create config object + +$config->define('version',{ ALIAS => 'V',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0}); +$config->define('help',{ ALIAS => 'h',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0}); +$config->define('man',{ ALIAS => 'm',ARGCOUNT => ARGCOUNT_NONE, ARGS => '!',DEFAULT => 0}); +$config->define('DBengine',{ VALIDATE => '[\w]+',DEFAULT => "Pg"}); +$config->define('DBname',{ VALIDATE => '[\w]+',DEFAULT => "hd_ob8xxx"}); +$config->define('DBhost',{ VALIDATE => '[-\w]+',DEFAULT => "dx1f"}); +$config->define('DBport',{ ARGS => '=i',DEFAULT => 5432}); +$config->define('master',{ VALIDATE => '[.\w]+',DEFAULT => "HGstation"}); +$config->define('basedir',{ VALIDATE => '[- /.\w]+',DEFAULT => $ENV{HYDROGENHOME} . "/bin"}); + +$config->file($ENV{HYDROGENHOME} . "/input/hg.cfg"); # look in user's $HYDROGENHOME to find configured settings +$config->args(\@ARGV); # get config settings from the command-line, overwriting any settings from the file... + +my $master = $config->get('master'); # name of DB table or view which holds master list of IDs for which MXD files are to be generated... +my $DBengine = $config->get('DBengine'); +my $DBname = $config->get('DBname'); +my $DBhost = $config->get('DBhost'); +my $DBport = $config->get('DBport'); +my $baseDir = `pwd`; +chomp $baseDir; +my $DBstr; +my $wildcard; + +#Open a database connection and get the list of LIDs from the IHFS DB +if($DBengine eq "Pg") { + $DBstr = "dbi:$DBengine:dbname=$DBname;host=$DBhost;port=$DBport"; + $wildcard = '%'; +} else { + $DBstr = "dbi:$DBengine:$DBname"; + $wildcard = '*'; +} + +my $dbh = DBI->connect("$DBstr",undef,undef,{ChopBlanks => 1}) or warn $DBI::errstr; +# creates the list of WFOs based on the HydroGen .xxx_backup files +# and builds the query to create the list of LIDs +my $wfo=`ls -a /awips/hydroapps/HydroGen/ | grep _backup | cut -c2-4`; +my $list_len=length $wfo; +my $num_wfos=$list_len/4; +my $index=1; +my $off=0; +my $wfoid=substr($wfo,$off,3); +my $wfoID=uc $wfoid; +my $wfo_query = "(location.hsa = \'$wfoID\'"; +while ($index < $num_wfos){ + $off+=4; + $wfoid=substr($wfo,$off,3); + $wfoID=uc $wfoid; + $wfo_query .= " or location.hsa = \'$wfoID\'"; + $index++; +} +$wfo_query .= ")"; + +#my $list_type="river"; +our $mw = MainWindow->new; +$mw->title('Ad-Hoc NRLDB Update'); + +my $lst_lab= $mw->Label(-text => 'Add any Unlisted Locations (comma-separated): '); +my $sql = "select distinct hgstation.lid,location.name,location.hsa from hgstation,location where hgstation.lid = location.lid and $wfo_query order by 3,1;"; + +# get the list of LIDs +my $qhw = $dbh->prepare("$sql") or warn $DBI::errstr; + +our @lid_list; # = ($wildcard); + +#get the data from the DB +get_results($qhw,\@lid_list); + +#set up a static array with the tables that are allowed for ad-hoc updates +#table_list is the actual name of the DB tables, while tabledesc is a friendlier description that is displayed to the user +our @table_list = ('location','riverstat','crest','floodstmt','hgstation','floodcat','lowwater'); +my @tabledesc = ('Location','Riverstat','Crest History','Impacts','HGstation','Flood Categories','Low Water'); + +$dbh->disconnect(); + +#manipulate the results of the lid/hsa/name query for better display +my @liddeschsa; +our @lidsend; +$index=0; +my $num_lids=scalar(@lid_list); +while ($index < $num_lids){ + my $line = $lid_list[$index]; + my @results = split('\|',$line); + #my $lid = $lid_list[$index]; + my $lid_lid = $results[0]; + my $lid_name = $results[1]; + my $lid_hsa = $results[2]; + push(@liddeschsa,"$lid_hsa | $lid_lid | $lid_name"); + push(@lidsend,$lid_lid); + $index++; +} + +# Create the GUI object +# Labels for the LID and table scroll boxes +my $misc_ent = $mw->Entry(); +my $label1 = $mw->Label(-text => 'HSA|LID|Location Name'); +my $label2 = $mw->Label(-text => 'Tables'); + +# Create the scroll boxes for the LIDs and tables +my $lb1 = $mw->Scrolled('Listbox', + -scrollbars => 'osoe',-width=>50, + -selectmode => 'multiple', -exportselection=>0); +my $lb2 = $mw->Scrolled('Listbox', + -scrollbars => 'osow',-width=>20, + -selectmode => 'multiple',-exportselection=>0); + +# Add the arrays that we want to display in the list boxes +$lb1->insert('end', @liddeschsa); +$lb2->insert('end', @tabledesc); + +# Create the buttons +my $exit = $mw->Button(-text => 'Exit', + -command => [$mw => 'destroy']); +my $send = $mw->Button(-text => 'Send', + -command => \&send_button); +my $show_log = $mw->Button(-text => 'Show Log', + -command => \&show_log); +my $update_list = $mw->Button(-text => 'Update List', -command => \&upd_list); +# create the label and text box for the last pdate window +my $status_box = $mw->Text(-width=>20, -height=>3); +my $lb_status = $mw->Label(-width=>20, -height=>3,-text=>"Last Ad-Hoc Update:"); +my $last_update = `cat $NRLDB_LOG/last_nrldb_update.txt`; + +$status_box->insert('end',"$last_update"); + +# Crate the GUI using grid to specify the physical locations of the objects +$label1->grid(-row=>1, -column=>1, -columnspan=>3) ; +$label2->grid(-row=>1, -column=>4) ; +$lb1->grid(-row=>2, -column=>1, -columnspan=>3, -sticky=>"ew") ;#pack; +$lb2->grid(-row=>2, -column=>4, -columnspan=>1, -sticky=>"w") ;#pack; +$lst_lab->grid(-row=>3, -column=>1, -columnspan=>1); +$misc_ent->grid(-row=>3, -column=>2); +$lb_status->grid(-row=>4, -column=>1); +$status_box->grid(-row=>4, -column=>2, -columnspan=>3, -sticky=>"ew"); +$send->grid(-row=>5, -column=>1) ;#pack; +$show_log->grid(-row=>5,-column=>2); +$exit->grid(-row=>5, -column=>4) ;#pack; + +MainLoop; + +# End of main +# + +# The Send button functionality function +sub send_button { + # Get the indices of the selected array items + my @LIDindex = $lb1->curselection; + my @Tableindex = $lb2->curselection; + my $index=1; + my $misc_lid = $misc_ent-> get(); + # build the lists of LIDs and tables + $tables = $table_list[$Tableindex[0]]; + my $numLIDs=@LIDindex; + print "numLIDs: $numLIDs\n"; + my $numTables=@Tableindex; + if ($numLIDs > 0){ + $lids = $lidsend[$LIDindex[0]]; + while ($index < $numLIDs){ + $lids .= "," . $lidsend[$LIDindex[$index]]; + $index++; + } + $lids .= "," . $misc_lid; + } else { + $lids=$misc_lid; + } + $index=1; + while ($index < $numTables){ + $tables .= "," . $table_list[$Tableindex[$index]]; + $index++; + } + + # Create the call to the script and execute it using system() + my $cmd = "${BIN_DIR}/send_nrldb_update.sh -table $tables -lid $lids > ${NRLDB_LOG}/send_nrldb_update.log\n"; + system($cmd); + + # Create a dialog box to inform the user that their data has been sent + my $dsend=$mw->Dialog(-title=>'Sent NRLDB Update',-buttons=>['OK']); + my $text_field="NRLDB Update Sent for LIDs: $lids \n and tables: $tables\n"; + my $box=$dsend->add('Label',-text=>"$text_field")->pack(-side => 'left',-fill => 'both',-expand => 1); + my $button = $dsend->Show; +} +# This subroutine, copied from Mark Fenbers bless program, takes a db query and returns an array of results +sub get_results +{ + my $qh = shift; + my $array = shift; + my $record; + + if(defined $qh) { + if($qh->execute(@_)) { + while($record = $qh->fetchrow_arrayref) { + foreach (@$record) { $_ = "" if ! defined $_; } + push @$array,(join '|',@$record); + } + } else { + warn $DBI::errstr; + } + } else { warn "unable to prepare query \"$sql\"\n"; } +} + +#This subroutine displays the log from the send script in the form of a dialog box +sub show_log +{ + use Tk::Dialog; + my $text_field=`cat ${NRLDB_LOG}/send_nrldb_update.log`; + my $d = $mw->Dialog(-title=>'Show Log',-buttons => ['OK']); + my $box=$d->add('Label',-text=>"$text_field")->pack(-side => 'left',-fill => 'both',-expand => 1); + my $button = $d->Show; +} + diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/nrldb.conf.baseline b/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/nrldb.conf.baseline new file mode 100755 index 0000000000..25556e51e9 --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/nrldb.conf.baseline @@ -0,0 +1,6 @@ +dbhost = "dx1f" +dbuser = "awips" +dbpass = "" +nrldb_host = "165.92.28.1" +site = "CCC" +dbname = "hd_ob92ccc" diff --git a/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/sed_script.txt b/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/sed_script.txt new file mode 100644 index 0000000000..99f27bad14 --- /dev/null +++ b/nativeLib/files.native/awipsShare/hydroapps/whfs/local/data/app/nrldb/sed_script.txt @@ -0,0 +1 @@ +s/'/\\'/g