From d3a048725a0d5a9358356308129c7de47c6ddfe7 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:40:32 -0700 Subject: [PATCH] Per #1546, move steps to install external libraries needed to install MET and tools needed to run unit tests into seperate Dockerfiles --- scripts/docker/Dockerfile | 130 +------------------------------- scripts/docker/Dockerfile.base | 133 +++++++++++++++++++++++++++++++++ scripts/docker/Dockerfile.test | 26 +++++++ 3 files changed, 162 insertions(+), 127 deletions(-) create mode 100644 scripts/docker/Dockerfile.base create mode 100644 scripts/docker/Dockerfile.test diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 330e0bbadd..0e2219be23 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM centos:7 +ARG MET_BASE_IMAGE=minimum + +FROM dtcenter/met-base:${MET_BASE_IMAGE} MAINTAINER John Halley Gotway # @@ -20,137 +22,11 @@ ENV MET_GIT_NAME ${SOURCE_BRANCH} ENV MET_GIT_URL https://github.com/dtcenter/MET ENV MET_DEVELOPMENT true -# -# Define the compilers. -# -ENV CC /usr/bin/gcc -ENV CXX /usr/bin/g++ -ENV FC /usr/bin/gfortran -ENV F77 /usr/bin/gfortran - -# -# Define package URL's. -# -ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz -ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z - -ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/v4.4.1.1.zip -ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz - -ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar -ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz - -# -# Install the required packages. -# -RUN yum -y update \ - && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ - libpng-devel jasper jasper-devel zlib zlib-devel \ - cairo-devel freetype-devel epel-release \ - hostname m4 make tar tcsh ksh time wget which \ - flex flex-devel bison bison-devel unzip \ - && yum -y install git g2clib-devel hdf5-devel.x86_64 gsl-devel \ - && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ - && yum -y install python3 python3-devel python3-pip \ - && pip3 install --upgrade pip \ - && python3 -m pip install numpy xarray netCDF4 - # # Set the working directory. # WORKDIR /met -# -# Setup the environment for interactive bash/csh container shells. -# -RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ - && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ - && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ - && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ - && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ - && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc \ - && echo export LD_LIBRARY_PATH=/usr/local/lib >> /etc/bashrc \ - && echo setenv LD_LIBRARY_PATH /usr/local/lib >> /etc/csh.cshrc -ENV LD_LIBRARY_PATH /usr/local/lib - -# -# Download and install BUFRLIB. -# -RUN mkdir -p /met/logs \ - && mkdir -p /met/external_libs/BUFRLIB \ - && cd /met/external_libs/BUFRLIB \ - && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ - && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ - && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ - && chmod +x preproc_patch.sh \ - && LOG_FILE=/met/logs/BUFRLIB_build.log \ - && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ - && ./preproc_patch.sh *.F > ${LOG_FILE} \ - && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ - && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ - && ar crv libbufr.a *.o >> ${LOG_FILE} \ - && rm -f /usr/local/lib/libbufr.a \ - && cp *.a /usr/local/lib \ - && cd /met/external_libs \ - && rm -rf BUFRLIB - -# -# Download and install NetCDF4 (C and C++). -# -RUN mkdir -p /met/external_libs/netcdf \ - && cd /met/external_libs/netcdf \ - && echo "Downloading netcdf-c-4.4.1.1 from ${NETCDF4C_URL}" \ - && wget ${NETCDF4C_URL} \ - && unzip v4.4.1.1.zip \ - && cd netcdf-c-4.4.1.1 \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_configure.log \ - && echo "Configuring netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_make_install.log \ - && echo "Compiling netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading from ${NETCDF4CXX_URL}" \ - && cd /met/external_libs/netcdf \ - && wget ${NETCDF4CXX_URL} \ - && tar -xzf v4.3.0.tar.gz \ - && cd netcdf-cxx4-4.3.0 \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_configure.log \ - && echo "Configuring netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && ./configure > ${LOG_FILE} \ - && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_make_install.log \ - && echo "Compiling netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && cd /met/external_libs \ - && rm -rf netcdf - -# -# Download and install HDF4 and HDFEOS. -# -RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ - && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/HDF4.2r3 \ - && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ - && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ - && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ - && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ - && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ - && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && echo "Downloading hdfeos from ${HDFEOS_URL}" \ - && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ - && cd /met/external_libs/hdfeos \ - && LOG_FILE=/met/logs/hdfeos_configure.log \ - && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ - && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ - && LOG_FILE=/met/logs/hdfeos_make_install.log \ - && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ - && make install > ${LOG_FILE} \ - && mkdir /usr/local/hdfeos/include \ - && cp include/*.h /usr/local/hdfeos/include/. \ - && cd /met/external_libs \ - && rm -rf HDF4.2r3 hdfeos - # # Download and install MET and GhostScript fonts. # Delete the MET source code for tagged releases matching "v"*. diff --git a/scripts/docker/Dockerfile.base b/scripts/docker/Dockerfile.base new file mode 100644 index 0000000000..fc7bb153b0 --- /dev/null +++ b/scripts/docker/Dockerfile.base @@ -0,0 +1,133 @@ +FROM centos:7 +MAINTAINER John Halley Gotway + +# +# Define the compilers. +# +ENV CC /usr/bin/gcc +ENV CXX /usr/bin/g++ +ENV FC /usr/bin/gfortran +ENV F77 /usr/bin/gfortran + +# +# Define package URL's. +# +ENV HDF4_URL http://www.hdfgroup.org/ftp/HDF/releases/HDF4.2r3/src/HDF4.2r3.tar.gz +ENV HDFEOS_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/HDF-EOS2.16v1.00.tar.Z + +ENV NETCDF4C_URL https://github.com/Unidata/netcdf-c/archive/v4.4.1.1.zip +ENV NETCDF4CXX_URL https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz + +ENV BUFRLIB_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/BUFRLIB_v10-2-3.tar +ENV GSFONT_URL https://dtcenter.ucar.edu/dfiles/code/METplus/MET/docker_data/ghostscript-fonts-std-8.11.tar.gz + +# +# Install the required packages. +# +RUN yum -y update \ + && yum -y install file gcc gcc-gfortran gcc-c++ glibc.i686 libgcc.i686 \ + libpng-devel jasper jasper-devel zlib zlib-devel \ + cairo-devel freetype-devel epel-release \ + hostname m4 make tar tcsh ksh time wget which \ + flex flex-devel bison bison-devel unzip \ + && yum -y install git g2clib-devel hdf5-devel.x86_64 gsl-devel \ + && yum -y install gv ncview wgrib wgrib2 ImageMagick ps2pdf \ + && yum -y install python3 python3-devel python3-pip \ + && pip3 install --upgrade pip \ + && python3 -m pip install numpy xarray netCDF4 + +# +# Set the working directory. +# +WORKDIR /met + +# +# Setup the environment for interactive bash/csh container shells. +# +RUN echo export MET_BASE=/usr/local/share/met >> /etc/bashrc \ + && echo setenv MET_BASE /usr/local/share/met >> /etc/csh.cshrc \ + && echo export MET_FONT_DIR=/usr/local/share/met/fonts >> /etc/bashrc \ + && echo setenv MET_FONT_DIR /usr/local/share/met/fonts >> /etc/csh.cshrc \ + && echo export RSCRIPTS_BASE=/usr/local/share/met/Rscripts >> /etc/bashrc \ + && echo setenv RSCRIPTS_BASE /usr/local/share/met/Rscripts >> /etc/csh.cshrc \ + && echo export LD_LIBRARY_PATH=/usr/local/lib >> /etc/bashrc \ + && echo setenv LD_LIBRARY_PATH /usr/local/lib >> /etc/csh.cshrc +ENV LD_LIBRARY_PATH /usr/local/lib + +# +# Download and install BUFRLIB. +# +RUN mkdir -p /met/logs \ + && mkdir -p /met/external_libs/BUFRLIB \ + && cd /met/external_libs/BUFRLIB \ + && echo "Downloading BUFRLIB from ${BUFRLIB_URL}" \ + && curl -SL ${BUFRLIB_URL} | tar xC /met/external_libs/BUFRLIB \ + && cat preproc.sh | sed 's/cpp /cpp -traditional-cpp /g' > preproc_patch.sh \ + && chmod +x preproc_patch.sh \ + && LOG_FILE=/met/logs/BUFRLIB_build.log \ + && echo "Compiling BUFRLIB and writing log file ${LOG_FILE}" \ + && ./preproc_patch.sh *.F > ${LOG_FILE} \ + && ${CC} -c -DUNDERSCORE *.c >> ${LOG_FILE} \ + && ${FC} -c -fno-second-underscore *.f >> ${LOG_FILE} \ + && ar crv libbufr.a *.o >> ${LOG_FILE} \ + && rm -f /usr/local/lib/libbufr.a \ + && cp *.a /usr/local/lib \ + && cd /met/external_libs \ + && rm -rf BUFRLIB + +# +# Download and install NetCDF4 (C and C++). +# +RUN mkdir -p /met/external_libs/netcdf \ + && cd /met/external_libs/netcdf \ + && echo "Downloading netcdf-c-4.4.1.1 from ${NETCDF4C_URL}" \ + && wget ${NETCDF4C_URL} \ + && unzip v4.4.1.1.zip \ + && cd netcdf-c-4.4.1.1 \ + && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_configure.log \ + && echo "Configuring netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ + && ./configure > ${LOG_FILE} \ + && LOG_FILE=/met/logs/netcdf-c-4.4.1.1_make_install.log \ + && echo "Compiling netcdf-c-4.4.1.1 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && echo "Downloading from ${NETCDF4CXX_URL}" \ + && cd /met/external_libs/netcdf \ + && wget ${NETCDF4CXX_URL} \ + && tar -xzf v4.3.0.tar.gz \ + && cd netcdf-cxx4-4.3.0 \ + && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_configure.log \ + && echo "Configuring netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ + && ./configure > ${LOG_FILE} \ + && LOG_FILE=/met/logs/netcdf-cxx4-4.3.0_make_install.log \ + && echo "Compiling netcdf-cxx4-4.3.0 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && cd /met/external_libs \ + && rm -rf netcdf + +# +# Download and install HDF4 and HDFEOS. +# +RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \ + && curl -SL ${HDF4_URL} | tar zxC /met/external_libs \ + && cd /met/external_libs/HDF4.2r3 \ + && LOG_FILE=/met/logs/HDF4.2r3_configure.log \ + && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \ + && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \ + && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \ + && mv -f Makefile_NEW mfhdf/hdiff/Makefile \ + && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \ + && echo "Compiling HDF4.2r3 and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && echo "Downloading hdfeos from ${HDFEOS_URL}" \ + && curl -SL ${HDFEOS_URL} | tar zxC /met/external_libs \ + && cd /met/external_libs/hdfeos \ + && LOG_FILE=/met/logs/hdfeos_configure.log \ + && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \ + && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \ + && LOG_FILE=/met/logs/hdfeos_make_install.log \ + && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \ + && make install > ${LOG_FILE} \ + && mkdir /usr/local/hdfeos/include \ + && cp include/*.h /usr/local/hdfeos/include/. \ + && cd /met/external_libs \ + && rm -rf HDF4.2r3 hdfeos diff --git a/scripts/docker/Dockerfile.test b/scripts/docker/Dockerfile.test new file mode 100644 index 0000000000..bc7963cb91 --- /dev/null +++ b/scripts/docker/Dockerfile.test @@ -0,0 +1,26 @@ +ARG MET_BASE_IMAGE=minimum + +FROM dtcenter/met-base:${MET_BASE_IMAGE} +MAINTAINER John Halley Gotway + +# +# Set the working directory. +# +WORKDIR /met + +# +# Download and install MET and GhostScript fonts. +# Delete the MET source code for tagged releases matching "v"*. +# +RUN echo "Installing tools needed for running MET unit tests..." \ + && echo "Installing Perl XML Parser..." \ + && yum makecache \ + && yum -y install perl-XML-Parser \ + && echo "Installing R..." \ + && yum -y install R \ + && echo "Installing R ncdf4 1.19..." \ + && wget https://cran.r-project.org/src/contrib/ncdf4_1.19.tar.gz \ + && R CMD INSTALL ncdf4_1.19.tar.gz \ + && echo "Installing NCO (for ncdiff)..." \ + && yum -y install nco \ + && echo "Finished installing unit test tools"