From bba7c7df635700036cf950ff2e7c9f7d6ae0f73f Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Sat, 9 Jun 2018 10:14:09 -0600 Subject: [PATCH] build script updates for 17.1.1-6 release --- build/README.md | 65 +++++++++++++++++++++++++++ build/build_rpms.sh | 92 ++++++++++++++++++++++++--------------- build/setup.sh | 31 ++++++++++--- {build => dist}/comps.xml | 0 4 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 build/README.md rename {build => dist}/comps.xml (100%) diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000000..aaa9955ac2 --- /dev/null +++ b/build/README.md @@ -0,0 +1,65 @@ +# Unidata AWIPS Build Notes + +## Build the AWIPS Development Environment Docker Container + +* https://hub.docker.com/r/unidata/awips-ade/tags/ + +We create and use the Docker image unidata/awips-ade to build the two RHEL binary distributions of AWIPS (RPMS). The directory **awips2-builds/build/awips-ade/** contains Dockerfiles for 64-bit EL6 and EL7 CentOS. + + ./build/awips-ade/docker_build.sh ${os_version} + +where **${os_version}** is either *el6* or *el7*. + +ADE Docker images will be named with the latest major release (17.1.1) and OS version + + docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + unidata/awips-ade 17.1.1-el6 77ea90df5919 16 min ago 4.13GB + unidata/awips-ade 17.1.1-el7 f030be21eda9 23 min ago 3.95GB + +## Build AWIPS RPMs + +Build individual AWIPS RPMs with the command + + ./build/setup.sh el7 ${rpm_name} + +for example: + + ./build/setup.sh el7 awips2-java + ./build/setup.sh el7 awips2-python + ./build/setup.sh el7 awips2-postgresql + +You can also build group aliases: + + ./build/setup.sh el7 buildEDEX + ./build/setup.sh el7 buildCAVE + ./build/setup.sh el7 buildLocalization + ./build/setup.sh el7 buildShapefiles + + +Finally, if no program name is given (e.g. `./build/setup.sh el7`), then ALL AWIPS RPMs and groups will be built, in order: + + su - awips -c "/bin/bash $buildsh -ade" + su - awips -c "/bin/bash $buildsh -python" + su - awips -c "/bin/bash $buildsh -qpid" + su - awips -c "/bin/bash $buildsh -server" + su - awips -c "/bin/bash $buildsh -database" + su - awips -c "/bin/bash $buildsh -edex" + su - awips -c "/bin/bash $buildsh -httpd" + su - awips -c "/bin/bash $buildsh -cave" + +(See `./build/build_rpms.sh` and `./rpms/build/x86_64/rpms.sh` for more insight.) + +## Yum Repository + +AWIPS RPMs are written to the directories `./dist/el6-dev/` and `./dist/el7-dev/`, and are packaged as a YUM repository with the commands + + repomanage -k1 --old dist/${os_version}-dev | xargs rm -e + createrepo -g ../comps.xml dist/${os_version}-dev + +Optionally, you can push the repo to your webserver with the command + + rsync --archive --delete dist/${os_version}-dev ${USER}@{WEBSERVER}:{$REMOTE_DIR} + + + diff --git a/build/build_rpms.sh b/build/build_rpms.sh index 232d34366f..4fe8ae9cbd 100755 --- a/build/build_rpms.sh +++ b/build/build_rpms.sh @@ -1,25 +1,44 @@ #!/bin/sh -xe -# Run rpmbuild scripts for awips -# type=$(rpm -qa awips2 | rev | cut -d "." -f3|rev) -type=$1 -if [ -z "$type" ]; then - echo "supply type (el6, el7)" +# +# Build Unidata AWIPS RPMs from source +# Author: mjames@ucar.edu +# + +# +# Require el6 or el7 be specified +# RPM name is optional (see below) +# +os_version=$1 +rpmname=$2 + +if [ -z "$os_version" ]; then + echo "supply os_version (el6, el7)" exit fi +# +# Set up AWIPS environment +# . /awips2/repo/awips2-builds/build/buildEnvironment.sh buildsh=$REPO/awips2-builds/build/build.sh pushd $REPO -# if not mounted to docker container, clone from github -if [ ! -d awips2-ncep ]; then git clone https://github.com/Unidata/awips2-ncep.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-goesr ]; then git clone https://github.com/Unidata/awips2-goesr.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-unidata ]; then git clone https://github.com/Unidata/awips2-unidata.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-core ]; then git clone https://github.com/Unidata/awips2-core.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-core-foss ]; then git clone https://github.com/Unidata/awips2-core-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-foss ]; then git clone https://github.com/Unidata/awips2-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-nws ]; then git clone https://github.com/Unidata/awips2-nws.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi -if [ ! -d awips2-rpm ]; then git clone https://github.com/Unidata/awips2-rpm.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +# +# If local source directories, exist, mount them to the +# container, otherwise clone the repo from github +# +if [ ! -d awips2-ncep ]; then git clone https://github.com/Unidata/awips2-ncep.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-goesr ]; then git clone https://github.com/Unidata/awips2-goesr.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-unidata ]; then git clone https://github.com/Unidata/awips2-unidata.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-core ]; then git clone https://github.com/Unidata/awips2-core.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-core-foss ]; then git clone https://github.com/Unidata/awips2-core-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-foss ]; then git clone https://github.com/Unidata/awips2-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-nws ]; then git clone https://github.com/Unidata/awips2-nws.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-rpm ]; then git clone https://github.com/Unidata/awips2-rpm.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi + +# +# AWIPS Static files are too large to host on github +# if [ ! -d awips2-static ]; then mkdir awips2-static cd awips2-static @@ -27,37 +46,38 @@ if [ ! -d awips2-static ]; then tar -xvf static.tar rm -rf static.tar fi +# +# If RPM name is given +# +if [ ! -z "$rpmname" ]; then -# to test the dockerized build procedure on a quick rpm -#su - awips -c "/bin/bash $buildsh -b awips2-ldm" - -# provides rpm name as arg (ex: ./build/setup.sh el7 awips2-python-jep) -prog=$2 -if [ ! -z "$prog" ]; then - frst="$(echo $prog | head -c 1)" + frst="$(echo $rpmname | head -c 1)" if [[ "$frst" = "-" ]]; then - su - awips -c "/bin/bash $buildsh $prog" + # If first character is a dash, then a build group alias was given + su - awips -c "/bin/bash $buildsh $rpmname" else - su - awips -c "/bin/bash $buildsh -b $prog" + su - awips -c "/bin/bash $buildsh -b $rpmname" fi + else - # Build all groups (in this order) - #su - awips -c "/bin/bash $buildsh -ade" - #su - awips -c "/bin/bash $buildsh -python" - #su - awips -c "/bin/bash $buildsh -qpid" - #su - awips -c "/bin/bash $buildsh -server" - #su - awips -c "/bin/bash $buildsh -database" - #su - awips -c "/bin/bash $buildsh -b awips2-ldm" + + # If RPM name is not given build all groups in this order + su - awips -c "/bin/bash $buildsh -ade" + su - awips -c "/bin/bash $buildsh -python" + su - awips -c "/bin/bash $buildsh -qpid" + su - awips -c "/bin/bash $buildsh -server" + su - awips -c "/bin/bash $buildsh -database" su - awips -c "/bin/bash $buildsh -edex" - #su - awips -c "/bin/bash $buildsh -httpd" - #su - awips -c "/bin/bash $buildsh -cave" - ls -al + su - awips -c "/bin/bash $buildsh -httpd" + su - awips -c "/bin/bash $buildsh -cave" + fi -# Manage RPMs +# Move RPMs to awips2-builds/dist if [ "$(ls -A ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/x86_64/)" ]; then - mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/x86_64/* /awips2/repo/awips2-builds/dist/${type}-dev/x86_64/ + mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/x86_64/* /awips2/repo/awips2-builds/dist/${os_version}-dev/x86_64/ fi if [ "$(ls -A ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/noarch/)" ]; then - mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/noarch/* /awips2/repo/awips2-builds/dist/${type}-dev/noarch/ + mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/noarch/* /awips2/repo/awips2-builds/dist/${os_version}-dev/noarch/ fi + diff --git a/build/setup.sh b/build/setup.sh index 2e57c7b027..bb6ea59c17 100755 --- a/build/setup.sh +++ b/build/setup.sh @@ -1,12 +1,23 @@ #!/bin/sh -xe +# +# Unidata AWIPS Build Setup Script +# Author: mjames@ucar.edu +# + +# +# Require el6 or el7 be specified +# if [ -z "$1" ]; then echo "supply type (el6, el7)" exit fi os_version=$1 +rpmname=$2 builds_dir=" -v `pwd`:/awips2/repo/awips2-builds:rw " -# if repos exist locally, mount rather than clone (see build_rpms.sh) +# +# If local source directories, exist, mount them to the container +# if [ -d /awips2/repo/awips2-static ]; then static_dir=" -v /awips2/repo/awips2-static:/awips2/repo/awips2-static " ;fi if [ -d /awips2/repo/awips2-rpm ]; then rpm_dir=" -v /awips2/repo/awips2-rpm:/awips2/repo/awips2-rpm " ;fi if [ -d /awips2/repo/awips2-core ]; then core_dir=" -v /awips2/repo/awips2-core:/awips2/repo/awips2-core " ;fi @@ -19,17 +30,27 @@ if [ -d /awips2/repo/awips2-unidata ]; then upc_dir=" -v /awips2/repo/awips if [ -d /awips2/repo/python-awips ]; then python_dir=" -v /awips2/repo/python-awips:/awips2/repo/python-awips " ;fi dirs=$builds_dir$static_dir$rpm_dir$core_dir$corefoss_dir$foss_dir$nws_dir$ncep_dir$goesr_dir$upc_dir$python_dir -# run -sudo docker run --entrypoint=/bin/bash --privileged -d -ti -e "container=docker" $dirs unidata/awips-ade:17.1.1-$os_version +# +# Run Docker AWIPS ADE Image +# +imgname=unidata/awips-ade +imgvers=17.1.1 +sudo docker run --entrypoint=/bin/bash --privileged -d -ti -e "container=docker" $dirs $imgname:$imgvers-$os_version dockerID=$(sudo docker ps | grep awips-ade | awk '{print $1}' | head -1 ) sudo docker logs $dockerID -sudo docker exec -ti $dockerID /bin/bash -xec "/awips2/repo/awips2-builds/build/build_rpms.sh $os_version $2"; +sudo docker exec -ti $dockerID /bin/bash -xec "/awips2/repo/awips2-builds/build/build_rpms.sh $os_version $rpmname"; sudo docker stop $dockerID sudo docker rm -v $dockerID +# +# Update/Recreate YUM Repository +# if [[ $(whoami) == "mjames" ]]; then # local build sudo chown -R mjames:ustaff dist/${os_version}-dev repomanage -k1 --old dist/${os_version}-dev | xargs rm -f - createrepo -g ../../build/comps.xml dist/${os_version}-dev + createrepo -g ../comps.xml dist/${os_version}-dev + # + # Push to web server + # rsync --archive --delete dist/${os_version}-dev tomcat@www:/web/content/repos/yum/ fi diff --git a/build/comps.xml b/dist/comps.xml similarity index 100% rename from build/comps.xml rename to dist/comps.xml