From 8613cb2086fcc83b3f4db8443faf2c646d12b717 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 May 2020 09:58:14 +0200 Subject: [PATCH 1/4] install EasyBuild as a module in base image + clean up GCC 9.3.0 image --- containers/Dockerfile.base-centos7 | 16 ++++++++++++++-- containers/Dockerfile.centos7-GCC-9.3.0 | 18 +----------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/containers/Dockerfile.base-centos7 b/containers/Dockerfile.base-centos7 index b3e81a5c..570603b8 100644 --- a/containers/Dockerfile.base-centos7 +++ b/containers/Dockerfile.base-centos7 @@ -4,11 +4,13 @@ ARG EB_VER=4.2.1 LABEL easybuild_version=${EB_VER} # install required packages (EPEL is required for Lmod) RUN yum install -y epel-release \ - && yum install -y bzip2 file gcc-c++ git Lmod make openssl-devel patch python3-pip unzip vim which + && yum install -y bzip2 file gcc-c++ git Lmod make openssl openssl-devel patch python3-pip unzip vim which rdma-core-devel # install EasBuild & archspec RUN pip3 install easybuild==${EB_VER} archspec # add 'easybuild' user RUN useradd -ms /bin/bash easybuild +# create /scratch & /easybuild directories +RUN mkdir /scratch && chown easybuild:easybuild /scratch && mkdir /easybuild && chown easybuild:easybuild /easybuild # define $EB_PYTHON to ensure EasyBuild runs with python3 ENV EB_PYTHON=python3 # Define the ENTRYPOINT for subsequent commands (CMD) @@ -19,5 +21,15 @@ SHELL ["/bin/bash", "-l", "-c"] CMD ["bash"] # switch to 'easybuild' user USER easybuild +# install EasyBuild as a module (& clean up /scratch) +RUN eb --install-latest-eb-release --prefix /scratch --installpath /easybuild && rm -rf /scratch/* # print info on Lmod & EasyBuild -RUN ml --version && which -a eb && eb --version & eb --show-system-info +RUN ml use /easybuild/modules/all && ml EasyBuild && ml --version && which -a eb && eb --version & eb --show-system-info +# remove globally installed EasyBuild, we can use the module going forward +USER root +RUN pip3 uninstall -y easybuild +# switch to 'easybuild' user +USER easybuild +WORKDIR /home/easybuild +# disable output buffering in Python, so we see EasyBuild output as it progresses +ENV PYTHONUNBUFFERED=TRUE diff --git a/containers/Dockerfile.centos7-GCC-9.3.0 b/containers/Dockerfile.centos7-GCC-9.3.0 index f3b62dcc..bdc18968 100644 --- a/containers/Dockerfile.centos7-GCC-9.3.0 +++ b/containers/Dockerfile.centos7-GCC-9.3.0 @@ -10,24 +10,8 @@ RUN export HOST_CPU_ARCH=$(python3 -c 'import archspec.cpu; print(archspec.cpu.h # use $CPU_ARCH to define label and set it as environment variable (to use in subsequent builds LABEL cpu_arch=${CPU_ARCH} ENV CPU_ARCH=${CPU_ARCH} -# for example: docker build --build-arg CPU_ARCH=haswell -# create /scratch & /easybuild directories -USER root -RUN mkdir /scratch && chown easybuild:easybuild /scratch -RUN mkdir /easybuild && chown easybuild:easybuild /easybuild -USER easybuild # use /easybuild for everything that should persist in the container (software, modules and sources) # everything else goes in /scratch, which will be cleaned out -ENV EASYBUILD_PREFIX=/scratch -ENV EASYBUILD_INSTALLPATH=/easybuild -ENV EASYBUILD_SOURCEPATH=/easybuild/sources -# disable output buffering in Python, so we see EasyBuild output as it progresses -ENV PYTHONUNBUFFERED=TRUE -# install software & accompanying modules -RUN eb GCC-9.3.0.eb --robot --trace \ +RUN eb --prefix /scratch --installpath /easybuild GCC-9.3.0.eb --robot --trace \ # clean up everything in /scratch && rm -rf /scratch/* -# finalize /easybuild, no changes can be made in /easybuild that persist! -VOLUME /easybuild -# make sure we start in /home/easybuild rather than / when shell in started in container -WORKDIR /home/easybuild From 0a7ee3876de135fce2fbbfa1671991cc60ffe9a0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 May 2020 10:00:58 +0200 Subject: [PATCH 2/4] clean up foss/2020a image, don't finalize /easybuild yet --- containers/Dockerfile.centos7-foss2020a | 7 ------- 1 file changed, 7 deletions(-) diff --git a/containers/Dockerfile.centos7-foss2020a b/containers/Dockerfile.centos7-foss2020a index 3c48c510..5c207601 100644 --- a/containers/Dockerfile.centos7-foss2020a +++ b/containers/Dockerfile.centos7-foss2020a @@ -1,12 +1,5 @@ FROM easybuilders/test:centos7-eb421-gcc930 -USER root -RUN yum install -y rdma-core-devel openssl -USER easybuild # install software & accompanying modules RUN (eb foss-2020a.eb --robot || tail -100 `eb --last-log`) \ # clean up everything in /scratch && rm -rf /scratch/* -# finalize /easybuild, no changes can be made in /easybuild that persist! -VOLUME /easybuild -# make sure we start in /home/easybuild rather than / when shell in started in container -WORKDIR /home/easybuild From 6aa7927998323ca560f3ca821fc28deb2ce4f5d8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 28 May 2020 10:36:44 +0200 Subject: [PATCH 3/4] break up long lines, move up defining $PYTHONUNBUFFERED --- containers/Dockerfile.base-centos7 | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/containers/Dockerfile.base-centos7 b/containers/Dockerfile.base-centos7 index 570603b8..257a42cc 100644 --- a/containers/Dockerfile.base-centos7 +++ b/containers/Dockerfile.base-centos7 @@ -10,7 +10,10 @@ RUN pip3 install easybuild==${EB_VER} archspec # add 'easybuild' user RUN useradd -ms /bin/bash easybuild # create /scratch & /easybuild directories -RUN mkdir /scratch && chown easybuild:easybuild /scratch && mkdir /easybuild && chown easybuild:easybuild /easybuild +RUN mkdir /scratch && chown easybuild:easybuild /scratch \ + && mkdir /easybuild && chown easybuild:easybuild /easybuild +# disable output buffering in Python, so we see EasyBuild output as it progresses +ENV PYTHONUNBUFFERED=TRUE # define $EB_PYTHON to ensure EasyBuild runs with python3 ENV EB_PYTHON=python3 # Define the ENTRYPOINT for subsequent commands (CMD) @@ -22,14 +25,18 @@ CMD ["bash"] # switch to 'easybuild' user USER easybuild # install EasyBuild as a module (& clean up /scratch) -RUN eb --install-latest-eb-release --prefix /scratch --installpath /easybuild && rm -rf /scratch/* +RUN eb --install-latest-eb-release --prefix /scratch --installpath /easybuild \ + && rm -rf /scratch/* # print info on Lmod & EasyBuild -RUN ml use /easybuild/modules/all && ml EasyBuild && ml --version && which -a eb && eb --version & eb --show-system-info +RUN ml use /easybuild/modules/all \ + && ml EasyBuild \ + && ml --version \ + && which -a eb \ + && eb --version \ + && eb --show-system-info # remove globally installed EasyBuild, we can use the module going forward USER root RUN pip3 uninstall -y easybuild # switch to 'easybuild' user USER easybuild WORKDIR /home/easybuild -# disable output buffering in Python, so we see EasyBuild output as it progresses -ENV PYTHONUNBUFFERED=TRUE From 0027aa16f81e4a473a7ba174cd45f3b3912128bf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 29 May 2020 09:59:52 +0200 Subject: [PATCH 4/4] allow customizing base image via build args --- containers/Dockerfile.base-centos7 | 10 ++++++++-- containers/Dockerfile.centos7-GCC-9.3.0 | 8 +++++++- containers/Dockerfile.centos7-foss2020a | 8 +++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/containers/Dockerfile.base-centos7 b/containers/Dockerfile.base-centos7 index 257a42cc..d0997cdc 100644 --- a/containers/Dockerfile.base-centos7 +++ b/containers/Dockerfile.base-centos7 @@ -1,5 +1,10 @@ -FROM centos:7.8.2003 -# Set default easybuild version +ARG IMG_REGISTRY=docker.io +ARG IMG_ORG=library +ARG IMG_NAME=centos +ARG IMG_TAG=7.8.2003 +# allow CI/CD to change all of the above +FROM ${IMG_REGISTRY}/${IMG_ORG}/${IMG_NAME}:${IMG_TAG} +# set default easybuild version ARG EB_VER=4.2.1 LABEL easybuild_version=${EB_VER} # install required packages (EPEL is required for Lmod) @@ -39,4 +44,5 @@ USER root RUN pip3 uninstall -y easybuild # switch to 'easybuild' user USER easybuild +# make sure we start in /home/easybuild rather than / when shell in started in container WORKDIR /home/easybuild diff --git a/containers/Dockerfile.centos7-GCC-9.3.0 b/containers/Dockerfile.centos7-GCC-9.3.0 index bdc18968..aa67067a 100644 --- a/containers/Dockerfile.centos7-GCC-9.3.0 +++ b/containers/Dockerfile.centos7-GCC-9.3.0 @@ -1,4 +1,10 @@ -FROM easybuilders/base:centos7-eb4.2.1 +ARG IMG_REGISTRY=docker.io +ARG IMG_ORG=easybuilders +ARG IMG_NAME=base +ARG IMG_TAG=centos7-eb4.2.1 +# allow CI/CD to change all of the above +FROM ${IMG_REGISTRY}/${IMG_ORG}/${IMG_NAME}:${IMG_TAG} + # CPU_ARCH is expected to be defined in the "docker build" command, # for example: docker build --build-arg CPU_ARCH=haswell ARG CPU_ARCH diff --git a/containers/Dockerfile.centos7-foss2020a b/containers/Dockerfile.centos7-foss2020a index 5c207601..331dfeb3 100644 --- a/containers/Dockerfile.centos7-foss2020a +++ b/containers/Dockerfile.centos7-foss2020a @@ -1,4 +1,10 @@ -FROM easybuilders/test:centos7-eb421-gcc930 +ARG IMG_REGISTRY=docker.io +ARG IMG_ORG=easybuilders +ARG IMG_NAME=test +ARG IMG_TAG=centos7-eb421-gcc930 +# allow CI/CD to change all of the above +FROM ${IMG_REGISTRY}/${IMG_ORG}/${IMG_NAME}:${IMG_TAG} + # install software & accompanying modules RUN (eb foss-2020a.eb --robot || tail -100 `eb --last-log`) \ # clean up everything in /scratch