Issue #1663 - awips2-database will no longer install site-specific databases.

- removed the sql scripts for the obsolete hydro databases
- the hydro databases will now be installed by the site-specific localization rpms
- the database imports and associated db files that are not actually part of localization
  will be purged by the localization rpm once it is finished with them

Change-Id: I631395db0688d858a280bcdc01aa8b70dc2eb1c6

Former-commit-id: 0ef2ee6e49 [formerly 4d3731bf6d] [formerly ab135933da] [formerly 722d068105 [formerly ab135933da [formerly e4181aff738a926738ccabcd2c09706670a43924]]]
Former-commit-id: 722d068105
Former-commit-id: 587033a1da78250dbb1cee2033955c635e17a488 [formerly 5ab931d15e]
Former-commit-id: ac0ef6e7b9

localization/localization.OAX/utility/common_static/site/OAX/hydro/db/hydroDatabases.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# this script is used to specify the damcat and ihfs
+# hydro databases that should be created during the
+# localization process provided that a local database
+# is present.
+
+export DAMCAT_DATABASE=dc_ob7oax
+export DAMCAT_SQL_DUMP=dc_ob7oax
+
+export IHFS_DATABASE=hd_ob92oax
+export IHFS_SQL_DUMP=hd_ob92oax

rpms/awips2.core/Installer.database/component.spec

@@ -67,7 +67,7 @@ cp -r %{_baseline_workspace}/${EXPECTED_PATH_TO_CONFIG}/${CONFIG_FILE_TO_INCLUDE
 
 # Copy The SQL Scripts That The Database RPM Will Need To The
 # Temporary Directory.
-DIRS_TO_COPY=('damcat' 'hmdb' 'migrated' 'setup' 'SHEF' 'vtec' 'ebxml' 'events')
+DIRS_TO_COPY=('hmdb' 'migrated' 'setup' 'vtec' 'ebxml' 'events')
 for dir in ${DIRS_TO_COPY[*]};
 do
    cp -r %{_baseline_workspace}/${PATH_TO_DDL}/${dir}/* \
@@ -210,15 +210,6 @@ function execute_initial_sql_script()
       > ${SQL_LOG} 2>&1
 }
 
-function update_createDamcat()
-{
-   echo ${AWIPS2_DATA_DIRECTORY} | sed 's/\//\\\//g' > .awips2_escape.tmp
-   AWIPS2_DATA_DIRECTORY_ESCAPED=`cat .awips2_escape.tmp`
-   rm -f .awips2_escape.tmp
-   perl -p -i -e "s/%{database_files_home}%/${AWIPS2_DATA_DIRECTORY_ESCAPED}/g" \
-      ${SQL_SHARE_DIR}/createDamcat.sql
-}
-
 function update_createEbxml()
 {
    echo ${AWIPS2_DATA_DIRECTORY} | sed 's/\//\\\//g' > .awips2_escape.tmp
@@ -285,13 +276,6 @@ execute_psql_sql_script ${SQL_SHARE_DIR}/fxatext.sql metadata
 # create the events schema
 execute_psql_sql_script ${SQL_SHARE_DIR}/createEventsSchema.sql metadata
 
-execute_psql_sql_script ${SQL_SHARE_DIR}/hd_ob83oax.sql metadata
-
-update_createDamcat
-execute_psql_sql_script ${SQL_SHARE_DIR}/createDamcat.sql postgres
-execute_psql_sql_script ${SQL_SHARE_DIR}/dcob7oax.sql dc_ob7oax
-execute_psql_sql_script ${SQL_SHARE_DIR}/populateDamcatDatabase.sql dc_ob7oax
-
 update_createHMDB
 su ${AWIPS_DEFAULT_USER} -c \
    "${SQL_SHARE_DIR}/createHMDB.sh ${PSQL_INSTALL} ${AWIPS_DEFAULT_PORT} ${AWIPS_DEFAULT_USER} ${SQL_SHARE_DIR} ${SQL_LOG}"
@@ -305,95 +289,6 @@ control_pg_ctl "stop"
 copy_addl_config
 
 %preun
-if [ "${1}" = "1" ]; then
-   exit 0
-fi
-
-# Ensure that our data directory exists.
-if [ ! -d /awips2/data ]; then
-   exit 0
-fi
-
-# We Need PostgreSQL  
-
-POSTGRESQL_INSTALL="/awips2/postgresql"
-PSQL_INSTALL="/awips2/psql"
-
-POSTMASTER="${POSTGRESQL_INSTALL}/bin/postmaster"
-if [ ! -f ${POSTMASTER} ]; then
-   exit 0
-fi
-PG_CTL="${POSTGRESQL_INSTALL}/bin/pg_ctl"
-if [ ! -f ${PG_CTL} ]; then
-   exit 0
-fi
-DROPDB="${POSTGRESQL_INSTALL}/bin/dropdb"
-if [ ! -f ${DROPDB} ]; then
-   exit 0
-fi
-PG_RESTORE="${POSTGRESQL_INSTALL}/bin/pg_restore"
-if [ ! -f ${PG_RESTORE} ]; then
-   exit 0
-fi
-PSQL="${PSQL_INSTALL}/bin/psql"
-if [ ! -f ${PSQL} ]; then
-   exit 0
-fi
-# Determine who owns the PostgreSQL Installation
-DB_OWNER=`ls -l /awips2/ | grep -w 'data' | awk '{print $3}'`
-# Our log file
-SQL_LOG="${RPM_INSTALL_PREFIX}/static/database.maps/maps.log"
-
-# Determine if PostgreSQL is running.
-I_STARTED_POSTGRESQL="N"
-su ${DB_OWNER} -c \
-   "${PG_CTL} status -D /awips2/data > /dev/null 2>&1"
-RC="$?"
-if [ ! "${RC}" = "0" ]; then
-   su ${DB_OWNER} -c \
-      "${POSTMASTER} -D /awips2/data > /dev/null 2>&1 &"
-   RC="$?"
-   if [ ! "${RC}" = "0" ]; then
-      echo "Failed To Start The PostgreSQL Server."
-      exit 1
-   fi
-   # Give PostgreSQL Time To Start.
-   sleep 10
-   I_STARTED_POSTGRESQL="Y"
-   echo ""
-fi
-
-# Drop Databases The Official PostgreSQL Way. The User Is Responsible
-# For Cleaning Up The Data Directory That Was Given To: 'initdb'.
-DBS_TO_DROP=( 'dc_ob7oax' 'fxatext' 'hd_ob83oax' 'hmdb' 'metadata' )
-TBS_TO_DROP=( 'damcat' 'hmdb' 'metadata' 'pgdata_ihfs' )
-
-# Attempt To Drop The Databases In Our List ...
-for db in ${DBS_TO_DROP[*]};
-do
-   echo "Dropping Database ... ${db}"
-   su ${DB_OWNER} -c \
-      "${DROPDB} -U awips ${db}"
-done
-
-# Attempt To Drop The Tablespaces In Our List ...
-for tb in ${TBS_TO_DROP[*]};
-do
-   TB_DIR=`${PSQL} -U awips -d postgres -c "\db" | grep ${tb} | awk '{print $5}'`
-   su ${DB_OWNER} -c \
-      "${PSQL} -U awips -d postgres -c \"DROP TABLESPACE ${tb}\""
-      
-   # remove the tablespace directory.
-   if [ -d "${TB_DIR}" ]; then
-      su ${DB_OWNER} -c "rmdir ${TB_DIR}"
-   fi
-done
-
-# Attempt To Stop PostgreSQL (If We Started It)
-if [ "${I_STARTED_POSTGRESQL}" = "Y" ]; then
-   su ${DB_OWNER} -c \
-      "${PG_CTL} stop -D /awips2/data"   
-fi
 
 %postun
 

rpms/awips2.core/Installer.localization/component.spec

@@ -60,116 +60,277 @@ fi
 %pre
 
 %post
-# determine if we include ffmp shapefiles
-edex_utility="/awips2/edex/data/utility"
-site_directory="${edex_utility}/edex_static/site/%{_localization_site}"
-ffmp_shp_directory="${site_directory}/shapefiles/FFMP"
-
-# if we do not, exit
-if [ ! -d ${ffmp_shp_directory} ]; then
-   exit 0
-fi
-
-# verify the both the basins and streams shapefile are present.
-if [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.shp ] ||
-   [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.shp ]; then
-   # if they are not, exit
-   exit 0
-fi
-
-# verify that the files the streams and basins shapefile depend on
-# are present.
-if [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.dbf ] ||
-   [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.shx ] ||
-   [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.dbf ] ||
-   [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.shx ]; then
-   # if they are not, exit
-   exit 0
-fi
-
-# shapefiles exist
-shapefile_import_log="shapefile_import-%{_localization_site}.log"
-log_file="/awips2/database/sqlScripts/share/sql/${shapefile_import_log}"
-if [ -f ${log_file} ]; then
-   /bin/rm -f ${log_file}
-fi
-/bin/touch ${log_file}
-chmod 666 ${log_file}
-
-# only import the shapefiles, if we are on the same machine as the db
+# only import the shapefiles and/or hydro databases, if we are on 
+# the same machine as the db.
 # verify the following exists:
 #   1) /awips2/data/maps
 #   2) /awips2/postgresql/bin/postmaster
 #   3) /awips2/postgresql/bin/pg_ctl
 #   4) /awips2/psql/bin/psql
 #   5) /awips2/database/sqlScripts/share/sql/maps/importShapeFile.sh
+#   6) /awips2/postgresql/bin/pg_restore
 if [ ! -d /awips2/data/maps ] ||
    [ ! -f /awips2/postgresql/bin/postmaster ] ||
    [ ! -f /awips2/postgresql/bin/pg_ctl ] ||
    [ ! -f /awips2/psql/bin/psql ] ||
-   [ ! -f /awips2/database/sqlScripts/share/sql/maps/importShapeFile.sh ]; then
+   [ ! -f /awips2/database/sqlScripts/share/sql/maps/importShapeFile.sh ] ||
+   [ ! -f /awips2/postgresql/bin/pg_restore ]; then
    # we are missing a file or directory, exit
    exit 0
 fi
 
-a2_postmaster="/awips2/postgresql/bin/postmaster"
-a2_pg_ctl="/awips2/postgresql/bin/pg_ctl"
-a2_shp_script="/awips2/database/sqlScripts/share/sql/maps/importShapeFile.sh"
+localization_db_log="localization_db-%{_localization_site}.log"
+log_file="/awips2/database/sqlScripts/share/sql/${localization_db_log}"
+if [ -f ${log_file} ]; then
+   /bin/rm -f ${log_file}
+fi
+/bin/touch ${log_file}
+chmod 666 ${log_file}
 
-echo "Importing the FFMP Shapefiles ... Please Wait."
-/bin/date > ${log_file}
-echo "Preparing to import the FFMP shapefiles ..." >> ${log_file}
-
-# retrieve the owner of the database
-DB_OWNER=`ls -l /awips2/ | grep -w 'data' | awk '{print $3}'`
-# determine if PostgreSQL is running
+edex_utility="/awips2/edex/data/utility"
 I_STARTED_POSTGRESQL="NO"
-echo "Determining if PostgreSQL is running ..." >> ${log_file}
-su ${DB_OWNER} -c \
-   "${a2_pg_ctl} status -D /awips2/data >> ${log_file} 2>&1"
-RC=$?
-echo "" >> ${log_file}
+POSTGRESQL_RUNNING="NO"
 
-# start PostgreSQL if it is not running as the user that owns data
-if [ ${RC} -eq 0 ]; then
-   echo "INFO: PostgreSQL is running." >> ${log_file}
-else
-   echo "Starting PostgreSQL as User: ${DB_OWNER} ..." >> ${log_file}
-   su ${DB_OWNER} -c \
-      "${a2_postmaster} -D /awips2/data >> ${log_file} 2>&1 &"
-   if [ $? -ne 0 ]; then
-      echo "FATAL: Failed to start PostgreSQL." >> ${log_file}
-      exit 0
+function prepare()
+{
+   if [ "${POSTGRESQL_RUNNING}" = "YES" ]; then
+      return 0
    fi
    
-   # give PostgreSQL time to start.
-   /bin/sleep 10
-   I_STARTED_POSTGRESQL="YES"
-fi
+   local a2_postmaster="/awips2/postgresql/bin/postmaster"
+   local a2_pg_ctl="/awips2/postgresql/bin/pg_ctl"
+   
+   # retrieve the owner of the database
+   DB_OWNER=`ls -l /awips2/ | grep -w 'data' | awk '{print $3}'`
+   
+   # determine if PostgreSQL is running
+   I_STARTED_POSTGRESQL="NO"
+   echo "Determining if PostgreSQL is running ..." >> ${log_file}
+   su ${DB_OWNER} -c \
+      "${a2_pg_ctl} status -D /awips2/data >> ${log_file} 2>&1"
+   RC=$?
+   echo "" >> ${log_file}
+   
+   # start PostgreSQL if it is not running as the user that owns data
+   if [ ${RC} -eq 0 ]; then
+      echo "INFO: PostgreSQL is running." >> ${log_file}
+   else
+      echo "Starting PostgreSQL as User: ${DB_OWNER} ..." >> ${log_file}
+      su ${DB_OWNER} -c \
+         "${a2_postmaster} -D /awips2/data >> ${log_file} 2>&1 &"
+      if [ $? -ne 0 ]; then
+         echo "FATAL: Failed to start PostgreSQL." >> ${log_file}
+         return 0
+      fi
+   
+      # give PostgreSQL time to start.
+      /bin/sleep 10
+      I_STARTED_POSTGRESQL="YES"
+   fi
+   POSTGRESQL_RUNNING="YES"
+   
+   return 0  
+}
 
-echo "" >> ${log_file}
-# import the shapefiles; log the output
+function restartPostgreSQL()
+{
+   if [ "${POSTGRESQL_RUNNING}" = "NO" ]; then
+      return 0
+   fi
+   
+   local a2_pg_ctl="/awips2/postgresql/bin/pg_ctl"
+   
+   # retrieve the owner of the database
+   DB_OWNER=`ls -l /awips2/ | grep -w 'data' | awk '{print $3}'`
+   
+   echo "Restarting PostgreSQL ..." >> ${log_file}
+   su ${DB_OWNER} -c \
+      "${a2_pg_ctl} restart -D /awips2/data" >> ${log_file}
+   sleep 20
+   echo "PostgreSQL restart complete ..." >> ${log_file}
+}
 
-# import the ffmp basins
-/bin/bash ${a2_shp_script} \
-   ${ffmp_shp_directory}/FFMP_aggr_basins.shp \
-   mapdata ffmp_basins 0.064,0.016,0.004,0.001 \
-   awips 5432 /awips2 >> ${log_file} 2>&1
-if [ $? -ne 0 ]; then
-   echo "FATAL: failed to import the FFMP basins." >> ${log_file}
-   exit 0
-fi
+function importShapefiles()
+{   
+   local site_directory="${edex_utility}/edex_static/site/%{_localization_site}"
+   
+   # determine if we include ffmp shapefiles
+   local ffmp_shp_directory="${site_directory}/shapefiles/FFMP"
+   
+   # if we do not, halt
+   if [ ! -d ${ffmp_shp_directory} ]; then
+      return 0
+   fi
+   
+   # shapefiles exist
+   
+   prepare
+   
+   # verify the both the basins and streams shapefile are present.
+   if [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.shp ] ||
+      [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.shp ]; then
+      # if they are not, exit
+      return 0
+   fi
+   
+   # verify that the files the streams and basins shapefile depend on
+   # are present.
+   if [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.dbf ] ||
+      [ ! -f ${ffmp_shp_directory}/FFMP_aggr_basins.shx ] ||
+      [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.dbf ] ||
+      [ ! -f ${ffmp_shp_directory}/FFMP_ref_sl.shx ]; then
+      # if they are not, exit
+      return 0
+   fi
+   
+   local a2_shp_script="/awips2/database/sqlScripts/share/sql/maps/importShapeFile.sh"
+   
+   echo "Importing the FFMP Shapefiles ... Please Wait."
+   /bin/date >> ${log_file}
+   echo "Preparing to import the FFMP shapefiles ..." >> ${log_file}   
+   
+   echo "" >> ${log_file}
+   # import the shapefiles; log the output
+   
+   # import the ffmp basins
+   /bin/bash ${a2_shp_script} \
+      ${ffmp_shp_directory}/FFMP_aggr_basins.shp \
+      mapdata ffmp_basins 0.064,0.016,0.004,0.001 \
+      awips 5432 /awips2 >> ${log_file} 2>&1
+   if [ $? -ne 0 ]; then
+      echo "FATAL: failed to import the FFMP basins." >> ${log_file}
+      return 0
+   fi
+   
+   # import the ffmp streams
+   /bin/bash ${a2_shp_script} \
+      ${ffmp_shp_directory}/FFMP_ref_sl.shp \
+      mapdata ffmp_streams 0.064,0.016,0.004,0.001 \
+      awips 5432 /awips2 >> ${log_file} 2>&1
+   if [ $? -ne 0 ]; then
+      echo "FATAL: failed to import the FFMP streams." >> ${log_file}
+      return 0
+   fi
+   
+   # indicate success
+   echo "INFO: The FFMP shapefiles were successfully imported." >> ${log_file}
+   return 0
+}
 
-# import the ffmp streams
-/bin/bash ${a2_shp_script} \
-   ${ffmp_shp_directory}/FFMP_ref_sl.shp \
-   mapdata ffmp_streams 0.064,0.016,0.004,0.001 \
-   awips 5432 /awips2 >> ${log_file} 2>&1
-if [ $? -ne 0 ]; then
-   echo "FATAL: failed to import the FFMP streams." >> ${log_file}
-   exit 0
-fi
+function removeHydroDbDirectory()
+{
+   # remove the hydro db directory since it is not officially part
+   # of the localization.
 
+   local site_directory="${edex_utility}/common_static/site/%{_localization_site}"
+   local hydro_db_directory="${site_directory}/hydro/db"
+   
+   if [ -d ${hydro_db_directory} ]; then
+      rm -rf ${hydro_db_directory}
+      if [ $? -ne 0 ]; then
+         echo "WARNING: Failed to remove hydro db directory from localization."
+         echo "         Please remove directory manually: ${hydro_db_directory}."
+      fi
+   fi
+   
+   return 0
+}
+
+function restoreHydroDb()
+{
+   local site_directory="${edex_utility}/common_static/site/%{_localization_site}"
+   
+   # determine if we include the hydro databases
+   local hydro_db_directory="${site_directory}/hydro/db"
+   
+   # if we do not, halt
+   if [ ! -d ${hydro_db_directory} ]; then
+      return 0
+   fi
+   
+   # hydro databases exist
+   prepare   
+   
+   # verify that the hydro database definition is present
+   if [ ! -f ${hydro_db_directory}/hydroDatabases.sh ]; then
+      return 0
+   fi
+   
+   # discover the hydro databases
+   source ${hydro_db_directory}/hydroDatabases.sh
+   
+   # ensure that the expected information has been provided
+   if [ "${DAMCAT_DATABASE}" = "" ] ||
+      [ "${DAMCAT_SQL_DUMP}" = "" ] ||
+      [ "${IHFS_DATABASE}" = "" ] ||
+      [ "${IHFS_SQL_DUMP}" = "" ]; then
+      echo "Sufficient information has not been provided for the Hydro Restoration!" \
+         >> ${log_file}
+      return 0
+   fi
+   
+   # ensure that the specified databases are available for import
+   if [ ! -f ${hydro_db_directory}/${DAMCAT_DATABASE} ] ||
+      [ ! -f ${hydro_db_directory}/${IHFS_DATABASE} ]; then
+      echo "The expected Hydro Database Exports are not present!" >> ${log_file}
+      return 0
+   fi
+   
+   # update pg_hba.conf
+   
+   local default_damcat="dc_ob7oax"
+   local default_ihfs="hd_ob83oax"
+   local pg_hba_conf="/awips2/data/pg_hba.conf"
+   
+   # update the entry for the damcat database
+   perl -p -i -e "s/${default_damcat}/${DAMCAT_DATABASE}/g" ${pg_hba_conf}
+   if [ $? -ne 0 ]; then
+      echo "Failed to update damcat database in ${pg_hba_conf}!" >> ${log_file}
+      return 0
+   fi
+   
+   # update the entry for the ihfs database
+   perl -p -i -e "s/${default_ihfs}/${IHFS_DATABASE}/g" ${pg_hba_conf}
+   if [ $? -ne 0 ]; then
+      echo "Failed to update ihfs database in ${pg_hba_conf}!" >> ${log_file}
+      return 0
+   fi
+   
+   # prepare PostgreSQL
+   restartPostgreSQL
+   
+   echo "Restoring the Hydro Databases ... Please Wait."
+   /bin/date >> ${log_file}
+   echo "Preparing to restore the Hydro databases ..." >> ${log_file}
+   
+   local a2_pg_restore="/awips2/postgresql/bin/pg_restore"
+   
+   # perform the restoration
+   echo "Restoring Database ${DAMCAT_DATABASE} ..." >> ${log_file}
+   ${a2_pg_restore} -U awips -C -d postgres ${hydro_db_directory}/${DAMCAT_DATABASE} \
+      >> ${log_file} 2>&1
+   # do not check the return code because any errors encountered during
+   # the restoration may cause the return code to indicate a failure even
+   # though the database was successfully restored.
+   
+   echo "" >> ${log_file} 
+   
+   echo "Restoring Database ${IHFS_DATABASE} ..." >> ${log_file}
+   ${a2_pg_restore} -U awips -C -d postgres ${hydro_db_directory}/${IHFS_DATABASE} \
+      >> ${log_file} 2>&1
+   # do not check the return code because any errors encountered during
+   # the restoration may cause the return code to indicate a failure even
+   # though the database was successfully restored.
+   
+   # indicate success
+   echo "INFO: The Hydro databases were successfully restored." >> ${log_file}
+}
+
+importShapefiles
+restoreHydroDb
+removeHydroDbDirectory
+
+a2_pg_ctl="/awips2/postgresql/bin/pg_ctl"
 # if we started PostgreSQL, shutdown PostgreSQL
 if [ "${I_STARTED_POSTGRESQL}" = "YES" ]; then
    echo "" >> ${log_file}
@@ -185,8 +346,6 @@ if [ "${I_STARTED_POSTGRESQL}" = "YES" ]; then
    fi
 fi
 
-# indicate success
-echo "INFO: The FFMP shapefiles were successfully imported." >> ${log_file}
 exit 0
 
 %preun

rpms/build/common/rpms.sh

@@ -183,6 +183,7 @@ function buildLocalizationRPMs()
       fi
       export LOCALIZATION_DIRECTORY="${localization}"
       export COMPONENT_NAME="awips2-localization-${site}"
+      export site=${site}
 
       echo "Building localization rpm for site: ${site}."
 
@@ -193,6 +194,7 @@ function buildLocalizationRPMs()
          --define '_component_name %(echo ${COMPONENT_NAME})' \
          --define '_baseline_workspace %(echo ${WORKSPACE})' \
          --define '_localization_directory %(echo ${LOCALIZATION_DIRECTORY})' \
+         --define '_localization_site %(echo ${site})' \
          --buildroot ${AWIPSII_BUILD_ROOT} \
          ${localization_SPECIFICATION}
       RC=$?