From b6ff6b7b3d3d67e060cfb359c0c7fb9e9914d5a3 Mon Sep 17 00:00:00 2001 From: mjames-upc Date: Fri, 10 Nov 2017 10:45:15 -0700 Subject: [PATCH] docker build for awips-ade for el6 and7 --- .travis.yml | 8 +-- build/awips-ade/Dockerfile.awips-ade.el6 | 33 ++++++++++++ build/awips-ade/Dockerfile.awips-ade.el7 | 33 ++++++++++++ build/build_rhel.sh | 56 ------------------- build/build_rpms.sh | 47 ++++++++++++++++ build/docker_build.sh | 10 ++++ build/package_deps.txt | 68 ++++++++++++++++++++++++ build/setup.sh | 46 ++++++++++------ 8 files changed, 224 insertions(+), 77 deletions(-) create mode 100644 build/awips-ade/Dockerfile.awips-ade.el6 create mode 100644 build/awips-ade/Dockerfile.awips-ade.el7 delete mode 100755 build/build_rhel.sh create mode 100755 build/build_rpms.sh create mode 100755 build/docker_build.sh create mode 100644 build/package_deps.txt diff --git a/.travis.yml b/.travis.yml index efce05f7b2..ab044a04e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ language: ruby env: matrix: - - OS_TYPE=centos OS_VERSION=centos6 - - OS_TYPE=centos OS_VERSION=centos7 + - OS_VERSION=el6 + - OS_VERSION=el7 services: - docker @@ -18,6 +18,6 @@ before_install: - echo 'DOCKER_OPTS="-H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock -s devicemapper"' | sudo tee /etc/default/docker > /dev/null - sudo service docker restart - sleep 5 - - sudo docker pull ${OS_TYPE}:${OS_VERSION} + - sudo docker pull unidata/awips-ade:${OS_VERSION} -script: build/setup.sh ${OS_TYPE} ${OS_VERSION} +script: build/setup.sh ${OS_VERSION} diff --git a/build/awips-ade/Dockerfile.awips-ade.el6 b/build/awips-ade/Dockerfile.awips-ade.el6 new file mode 100644 index 0000000000..1a898ee86e --- /dev/null +++ b/build/awips-ade/Dockerfile.awips-ade.el6 @@ -0,0 +1,33 @@ +FROM centos:6 +ENV VERSION 17.1.1 +ENV RELEASE 4 +MAINTAINER Michael James + +USER root + +RUN yum update yum + +RUN groupadd fxalpha && useradd -G fxalpha awips + +RUN yum groupinstall "Development tools" -y +RUN yum install epel-release -y +RUN yum clean all -y + +ENV systemDeps="wget rsync git net-tools bc" +ENV rpmDeps="gcc gcc-c++ rpm-build readline-devel createrepo" +ENV qpidDeps="boost-devel cmake make ruby libuuid-devel" +ENV pythonDeps="tk-devel tcl-devel atlas-devel compat-libf2c-34(x86-64) libgfortran(x86-64) geos libpng-devel freetype" +ENV awipsDeps="netcdf netcdf-devel hdf5-devel lzo-devel bzip2-devel qt-devel" +ENV httpDeps="autoconf findutils libselinux-devel libxml2-devel lua-devel openldap-devel openssl-devel pcre-devel pkgconfig perl zlib-devel apr-util-devel apr-devel" + +RUN yum install $systemDeps $rpmDeps $qpidDeps $pythonDeps $awipsDeps $httpDeps -y + +RUN wget -O /etc/yum.repos.d/awips2.repo http://www.unidata.ucar.edu/software/awips2/doc/awips-dev.repo +RUN yum -y clean all +RUN yum groupinstall awips2-ade -y + +RUN mkdir -p /awips2/jenkins/buildspace/workspace/AWIPS2-UPC_build/baseline && mkdir -p /awips2/jenkins/buildspace/workspace/tmp +RUN mkdir -p /awips2/jenkins/build/rpms/awips2_${VERSION}/{x86_64,noarch}/ +RUN chown -R awips:fxalpha /awips2/jenkins/ + +ENTRYPOINT ["/bin/bash"] diff --git a/build/awips-ade/Dockerfile.awips-ade.el7 b/build/awips-ade/Dockerfile.awips-ade.el7 new file mode 100644 index 0000000000..ebba5c677e --- /dev/null +++ b/build/awips-ade/Dockerfile.awips-ade.el7 @@ -0,0 +1,33 @@ +FROM centos:7 +ENV VERSION 17.1.1 +ENV RELEASE 4 +MAINTAINER Michael James + +USER root + +RUN yum update yum + +RUN groupadd fxalpha && useradd -G fxalpha awips + +RUN yum groupinstall "Development tools" -y +RUN yum install epel-release -y +RUN yum clean all -y + +ENV systemDeps="wget rsync git net-tools bc" +ENV rpmDeps="gcc gcc-c++ rpm-build readline-devel createrepo" +ENV qpidDeps="boost-devel cmake make ruby libuuid-devel" +ENV pythonDeps="tk-devel tcl-devel atlas-devel compat-libf2c-34(x86-64) libgfortran(x86-64) geos libpng-devel freetype" +ENV awipsDeps="netcdf netcdf-devel hdf5-devel lzo-devel bzip2-devel qt-devel" +ENV httpDeps="autoconf findutils libselinux-devel libxml2-devel lua-devel openldap-devel openssl-devel pcre-devel pkgconfig perl zlib-devel apr-util-devel apr-devel" + +RUN yum install $systemDeps $rpmDeps $qpidDeps $pythonDeps $awipsDeps $httpDeps -y + +RUN wget -O /etc/yum.repos.d/awips2.repo http://www.unidata.ucar.edu/software/awips2/doc/el7-dev.repo +RUN yum -y clean all +RUN yum groupinstall awips2-ade -y + +RUN mkdir -p /awips2/jenkins/buildspace/workspace/AWIPS2-UPC_build/baseline && mkdir -p /awips2/jenkins/buildspace/workspace/tmp +RUN mkdir -p /awips2/jenkins/build/rpms/awips2_${VERSION}/{x86_64,noarch}/ +RUN chown -R awips:fxalpha /awips2/jenkins/ + +ENTRYPOINT ["/bin/bash"] diff --git a/build/build_rhel.sh b/build/build_rhel.sh deleted file mode 100755 index 348a55a8c5..0000000000 --- a/build/build_rhel.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -xe -OS_TYPE=$1 -OS_VERSION=$2 -if [ $OS_VERSION == "centos7" ]; then - REPO_EXT="-el7" - repo_name="el7.repo" -else - repo_name="awips-dev.repo" -fi - -groupadd fxalpha && useradd -G fxalpha awips -yum -y clean all -yum -y clean expire-cache -yum groupinstall "Development tools" -y >& /dev/null -yum install wget qt-devel cmake gcc gcc-c++ ruby boost-devel libuuid-devel tk-devel tcl-devel rsync git net-tools rpm-build netcdf-devel -y -wget -O /etc/yum.repos.d/awips2.repo http://www.unidata.ucar.edu/software/awips2/doc/${repo_name} -yum -y clean all -yum groupinstall awips2-ade -y - -. /awips2/repo/awips2-src/rpms/unidata/buildEnvironment.sh - -mkdir -p /awips2/jenkins/buildspace/workspace/AWIPS2-UPC_build/baseline -mkdir -p /awips2/jenkins/buildspace/workspace/tmp -mkdir -p /awips2/jenkins/build/rpms/awips2_${AWIPSII_VERSION}/{x86_64,noarch}/ - -pushd /awips2/repo - -git clone https://github.com/Unidata/awips2.git --branch unidata_${AWIPSII_VERSION}${REPO_EXT} --single-branch awips2-builds -git clone https://github.com/Unidata/awips2-ncep.git --branch unidata_${AWIPSII_VERSION} --single-branch -git clone https://github.com/Unidata/awips2-core.git --branch unidata_${AWIPSII_VERSION} --single-branch -git clone https://github.com/Unidata/awips2-core-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch -git clone https://github.com/Unidata/awips2-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch -git clone https://github.com/Unidata/awips2-rpmbuild.git awips2-rpm --branch unidata_${AWIPSII_VERSION} --single-branch -git clone https://github.com/Unidata/awips2-nws.git --branch unidata_${AWIPSII_VERSION} --single-branch - -cd /awips2/repo/awips2-builds/rpms/unidata/ - -# build all RPMs -/bin/bash cibuild.sh -qpid -/bin/bash cibuild.sh -python -/bin/bash cibuild.sh -server -/bin/bash cibuild.sh -database -/bin/bash cibuild.sh -dev -/bin/bash cibuild.sh -b buildEDEX >& /dev/null -/bin/bash cibuild.sh -b buildCAVE >& /dev/null - -export RPMDIR=/awips2/jenkins/build/rpms/awips2_${AWIPSII_VERSION}/ -export LD_LIBRARY_PATH=/usr/lib:/lib:/usr/lib64:/lib64 -cp comps.xml ${RPMDIR} -cd ${RPMDIR} -repomanage -k1 --old . | xargs rm -f -createrepo -g ./comps.xml . -unset LD_LIBRARY_PATH -. /etc/profile.d/awips2.sh - -tar -cf /awips2/jenkins/build/rpms/awips2_${AWIPSII_VERSION}/ /awips2/repo/awips2-src/dist/awips2_${AWIPSII_VERSION}_${OS_TYPE}-${OS_VERSION}.tar diff --git a/build/build_rpms.sh b/build/build_rpms.sh new file mode 100755 index 0000000000..f0059dc9fb --- /dev/null +++ b/build/build_rpms.sh @@ -0,0 +1,47 @@ +#!/bin/sh -xe +# Run rpmbuild scripts for awips +# type=$(rpm -qa awips2 | rev | cut -d "." -f3|rev) +type=$1 +if [ -z "$type" ]; then + echo "supply type (el6, el7)" + exit +fi + +. /awips2/repo/awips2-builds/rpms/unidata/buildEnvironment.sh +buildsh=$REPO/awips2-builds/rpms/unidata/cibuild.sh +pushd $REPO + +# if not mounted to docker container, clone from github +if [ ! -d awips2-ncep ]; then git clone https://github.com/Unidata/awips2-ncep.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-core ]; then git clone https://github.com/Unidata/awips2-core.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-core-foss ]; then git clone https://github.com/Unidata/awips2-core-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-foss ]; then git clone https://github.com/Unidata/awips2-foss.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-nws ]; then git clone https://github.com/Unidata/awips2-nws.git --branch unidata_${AWIPSII_VERSION} --single-branch ;fi +if [ ! -d awips2-rpm ]; then git clone https://github.com/Unidata/awips2-rpmbuild.git awips2-rpm --branch el7_rpm --single-branch ;fi +if [ ! -d awips2-static ]; then + mkdir awips2-static + cd awips2-static + wget http://www.unidata.ucar.edu/downloads/awips2/static.tar + tar -xvf static.tar + rm -rf static.tar +fi + +# Test build procedure on a quick rpm +su - awips -c "/bin/bash $buildsh -b awips2-ldm" + +# Build all groups (in this order) +#su - awips -c "/bin/bash $buildsh -ade" +#su - awips -c "/bin/bash $buildsh -python" +#su - awips -c "/bin/bash $buildsh -qpid" +#su - awips -c "/bin/bash $buildsh -server" +#su - awips -c "/bin/bash $buildsh -edex" +#su - awips -c "/bin/bash $buildsh -database" +#su - awips -c "/bin/bash $buildsh -cave" + +# Manage RPMs +if [ "$(ls -A ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/x86_64/)" ]; then + mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/x86_64/* /awips2/repo/awips2-builds/dist/$type/x86_64/ +fi +if [ "$(ls -A ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/noarch/)" ]; then + mv ${JENKINS_HOME}/build/rpms/awips2_${AWIPSII_VERSION}/noarch/* /awips2/repo/awips2-builds/dist/$type/noarch/ +fi diff --git a/build/docker_build.sh b/build/docker_build.sh new file mode 100755 index 0000000000..389dd01418 --- /dev/null +++ b/build/docker_build.sh @@ -0,0 +1,10 @@ +#!/bin/bash +existing=$(sudo docker images |grep awips-ade | awk '{ print $3 }'|uniq) +if [ ! -z "$existing" ]; then + sudo docker rmi $existing +fi +pushd /awips2/repo/awips2-builds/build/awips-ade +sudo docker build -t unidata/awips-ade -f Dockerfile.awips-ade.el7 . +dockerID=$(sudo docker ps | grep awips-ade | awk '{print $1}' | head -1 ) +sudo docker tag $dockerID unidata/awips-ade:17.1.1-el7 +sudo docker push unidata/awips-ade diff --git a/build/package_deps.txt b/build/package_deps.txt new file mode 100644 index 0000000000..d5c0a88d7e --- /dev/null +++ b/build/package_deps.txt @@ -0,0 +1,68 @@ +# All BuildRequires +# httpd-pypies autoconf +# httpd-pypies findutils +# httpd-pypies libselinux-devel +# httpd-pypies libuuid-devel +# httpd-pypies libxml2-devel +# httpd-pypies lua-devel +# httpd-pypies openldap-devel +# httpd-pypies openssl-devel +# httpd-pypies pcre-devel +# httpd-pypies perl +# httpd-pypies pkgconfig +# httpd-pypies zlib-devel +# postgresql openssl-devel +# qpid-lib boost-devel +# qpid-lib cmake +# qpid-lib libuuid-devel +# qpid-lib make +# qpid-lib ruby +# python tk-devel +# python tcl-devel +# numpy atlas-devel +# numpy compat-libf2c-34(x86-64) +# numpy libgfortran(x86-64 +# h5py awips2-python-numpy +# h5py awips2-python-cython +# scientific netcdf +# shapely geos +# matplotlib libpng +# matplotlib freetype +# notification libuuid-devel + +# All Requires +# cave openmotif +# cave libMrm.so.4()(64bit) +# cave libXp.so.6()(64bit) +# cave libg2c.so.0()(64bit) +# common-base netcdf +# common-base netcdf-devel +# database libpng +# database netcdf +# database netcdf-devel +# gfesuite numpy +# notification boost >= 1.33.1 +# awips wget +# awips unzip +# awips bc +# awips ksh +# awips libstdc++(x86-32) +# awips libgfortran(x86-32) +# ldm pax +# ldm gcc +# ldm libxml2-devel +# ldm boost-devel +# ldm libtool +# ldm libpng-devel +# h5py libz.so.1 +# httpd-pypies initscripts >= 8.36, /etc/mime.types +# httpd-pypies pkgconfig, libtool +# jep atlas +# matplotlib libpng, freetype +# numpy compat-libf2c-34(x86-64) >= 3.4.6-19.el6 +# numpy libgfortran(x86-64) >= 4.4.7-3.el6 +# postgresql openssl >= 1.0.1e +# postgresql netcdf, geos +# qpid-java log4j >= 1.2.12 +# scientific netcdf + diff --git a/build/setup.sh b/build/setup.sh index af25434183..eb5d7e1e65 100755 --- a/build/setup.sh +++ b/build/setup.sh @@ -1,19 +1,31 @@ #!/bin/sh -xe -os_type=$1 -os_version=$2 - -if [ "$os_version" = "centos6" ]; then - - sudo docker run --rm=true -v `pwd`:/awips2/repo/awips2-src:rw ${os_type}:${os_version} /bin/bash -c "bash -xe /awips2/repo/awips2-src/build/build_rhel.sh ${os_type} ${os_version}" - -elif [ "$os_version" = "centos7" ]; then - - sudo docker run --privileged -d -ti -e "container=docker" -v `pwd`:/awips2/repo/awips2-src:rw ${os_type}:${os_version} /usr/sbin/init - DOCKER_CONTAINER_ID=$(sudo docker ps | grep ${os_version} | awk '{print $1}' | head -1 ) - sudo docker logs $DOCKER_CONTAINER_ID - sudo docker exec -ti $DOCKER_CONTAINER_ID /bin/bash -xec "bash -xe /awips2/repo/awips2-src/build/build_rhel.sh ${os_type} ${os_version}"; - sudo docker ps -a - sudo docker stop $DOCKER_CONTAINER_ID - sudo docker rm -v $DOCKER_CONTAINER_ID - +if [ -z "$1" ]; then + echo "supply type (el6, el7)" + exit +fi +os_version=$1 +builds_dir=" -v `pwd`:/awips2/repo/awips2-builds:rw " + +# if repos exist locally, mount rather than clone (see build_rpms.sh) +if [ -d /awips2/repo/awips2-static ]; then static_dir=" -v /awips2/repo/awips2-static:/awips2/repo/awips2-static " ;fi +if [ -d /awips2/repo/awips2-rpm ]; then rpm_dir=" -v /awips2/repo/awips2-rpm:/awips2/repo/awips2-rpm " ;fi +if [ -d /awips2/repo/awips2-core ]; then core_dir=" -v /awips2/repo/awips2-core:/awips2/repo/awips2-core " ;fi +if [ -d /awips2/repo/awips2-core-foss ]; then corefoss_dir=" -v /awips2/repo/awips2-core-foss:/awips2/repo/awips2-core-foss " ;fi +if [ -d /awips2/repo/awips2-foss ]; then foss_dir=" -v /awips2/repo/awips2-foss:/awips2/repo/awips2-foss " ;fi +if [ -d /awips2/repo/awips2-nws ]; then nws_dir=" -v /awips2/repo/awips2-nws:/awips2/repo/awips2-nws " ;fi +if [ -d /awips2/repo/awips2-ncep ]; then ncep_dir=" -v /awips2/repo/awips2-ncep:/awips2/repo/awips2-ncep " ;fi +if [ -d /awips2/repo/python-awips ]; then python_dir=" -v /awips2/repo/python-awips:/awips2/repo/python-awips " ;fi +dirs=$builds_dir$static_dir$rpm_dir$core_dir$corefoss_dir$foss_dir$nws_dir$ncep_dir$python_dir + +# run +sudo docker run --entrypoint=/bin/bash --privileged -d -ti -e "container=docker" $dirs unidata/awips-ade:17.1.1-$os_version +dockerID=$(sudo docker ps | grep awips-ade | awk '{print $1}' | head -1 ) +sudo docker logs $dockerID +sudo docker exec -ti $dockerID /bin/bash -xec "/awips2/repo/awips2-builds/build/build_rpms.sh $os_version"; +sudo docker stop $dockerID +sudo docker rm -v $dockerID + +if [[ $USER == "mjames" ]]; then # local build + sudo chown -R mjames:ustaff dist/$os_version + createrepo -g ../../rpms/unidata/comps.xml dist/$os_version fi