diff options
Diffstat (limited to 'deployment')
-rw-r--r-- | deployment/docker/Dockerfile | 71 | ||||
-rw-r--r-- | deployment/docker/Dockerfile_alma9 | 90 | ||||
-rw-r--r-- | deployment/docker/Dockerfile_centos7 | 133 | ||||
-rw-r--r-- | deployment/docker/Dockerfile_fedora | 79 | ||||
-rw-r--r-- | deployment/docker/Dockerfile_ubuntu | 80 | ||||
-rw-r--r-- | deployment/gitlab/config.toml | 16 |
6 files changed, 469 insertions, 0 deletions
diff --git a/deployment/docker/Dockerfile b/deployment/docker/Dockerfile new file mode 100644 index 00000000..4dd9ab8a --- /dev/null +++ b/deployment/docker/Dockerfile @@ -0,0 +1,71 @@ +FROM debian:buster + +RUN apt-get update && apt-get install -y \ + pkg-config \ + cmake \ + build-essential \ + libhdf5-dev \ + libgsl-dev \ + libgtk-3-dev \ + libcairo2-dev \ + libeigen3-dev \ + libpango1.0-dev \ + libgdk-pixbuf2.0-dev \ + libfftw3-dev \ + libncurses-dev \ + libpng-dev \ + libtiff5-dev \ + git \ + flex \ + bison \ + libzmq3-dev \ + libmsgpack-dev \ + python3-dev \ + python3-pip \ + unzip \ + wget \ + curl \ + ninja-build \ + gfortran \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +WORKDIR /root +RUN pip3 install meson + +WORKDIR /home/crystfel-build + +# Mosflm +RUN wget -nv https://www.mrc-lmb.cam.ac.uk/mosflm/mosflm/ver740/pre-built/mosflm-linux-64-noX11.zip +RUN unzip mosflm-linux-64-noX11.zip +RUN mv mosflm-linux-64-noX11 /usr/local/bin/mosflm + +# CrystFEL +RUN git clone https://gitlab.desy.de/thomas.white/crystfel.git +RUN cd crystfel && meson build -Dprefix=/usr/local +RUN cd crystfel && ninja -C build +RUN cd crystfel && ninja -C build test +RUN cd crystfel && ninja -C build install + +## Stage 2 +FROM debian:buster-slim +RUN apt-get update && apt-get install -y \ + libhdf5-103 \ + libgsl23 \ + libgtk-3-0 \ + libcairo2 \ + libpango1.0 \ + libgdk-pixbuf2.0 \ + libfftw3-double3 \ + libncurses6 \ + libpng16-16 \ + libtiff5 \ + libzmq5 \ + libmsgpackc2 \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +COPY --from=0 /usr/local /usr/local + +# Environment variable needed for CrystFEL GUI and Mosflm +# The file is installed by libccp4c, a wrapped subproject of CrystFEL +ENV SYMINFO=/usr/share/ccp4/syminfo.lib + +RUN ldconfig diff --git a/deployment/docker/Dockerfile_alma9 b/deployment/docker/Dockerfile_alma9 new file mode 100644 index 00000000..38df2b30 --- /dev/null +++ b/deployment/docker/Dockerfile_alma9 @@ -0,0 +1,90 @@ +FROM almalinux:9.4 + +ENV CFPREFIX=/software/crystfel/devel +ENV PKG_CONFIG_PATH=$CFPREFIX/lib64/pkgconfig:$CFPREFIX/lib/pkgconfig +ENV LD_LIBRARY_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV LD_RUN_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV PATH=$CFPREFIX/bin:$PATH + +RUN dnf -y install epel-release +RUN crb enable +RUN dnf -y update +RUN dnf -y group install 'Development Tools' +RUN dnf -y install gsl-devel flex bison gcc-c++ gcc-gfortran gtk3-devel cairo-devel \ + pango-devel gdk-pixbuf2-devel fftw-devel libpng-devel diffutils \ + zeromq-devel python3-devel which wget lz4 lz4-devel \ + bzip2-libs bzip2-devel libcurl-devel openssh-clients krb5-workstation cmake \ + meson ninja-build pandoc +RUN sed -i -E '/default_ccache_name/s//# default_ccache_name/g' /etc/krb5.conf ; \ + sed -i -E '/#\s+default_realm\s* =\s*EXAMPLE.COM\s*/s//default_realm = DESY.DE/g' /etc/krb5.conf ; \ + if [ ! -d ~/.ssh ]; then \ + mkdir -p ~/.ssh; \ + chmod 700 ~/.ssh; \ + fi; \ + wget -q https://wims.desy.de/system/ALL_afs/etc/ssh_known_hosts2 -O ~/.ssh/known_hosts + +# HDF5 +RUN wget -nv https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/src/hdf5-1.12.1.tar.bz2; \ + tar -xf hdf5-1.12.1.tar.bz2; \ + pushd hdf5-1.12.1; \ + ./configure --prefix=$CFPREFIX \ + --enable-shared --disable-static \ + --with-default-plugindir=${CFPREFIX}/lib64/plugins; \ + make; \ + make install; \ + popd # hdf5-1.12.1 + +# HDF5 filter plugins +RUN git clone --depth 1 https://github.com/nexusformat/HDF5-External-Filter-Plugins.git ; \ + pushd HDF5-External-Filter-Plugins; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DENABLE_BITSHUFFLE_PLUGIN=yes \ + -DENABLE_LZ4_PLUGIN=yes \ + -DENABLE_BZIP2_PLUGIN=yes \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd; \ + make -C build -j 32; \ + make -C build install; \ + popd # HDF5-External-Filter-Plugins + +# MsgPack +RUN wget -nv https://github.com/msgpack/msgpack-c/releases/download/cpp-3.3.0/msgpack-3.3.0.tar.gz ; \ + tar -xzf msgpack-3.3.0.tar.gz ; \ + pushd msgpack-3.3.0; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DMSGPACK_BUILD_EXAMPLES=OFF \ + -DMSGPACK_ENABLE_CXX=OFF \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd ; \ + make -C build -j 32 ; \ + make -C build install ; \ + popd # msgpack-3.3.0 + +# ASAP::O +RUN git clone https://gitlab.desy.de/asapo/asapo.git ; \ + pushd asapo ; \ + git checkout 23.12.0-alpha ; \ + mkdir build ; \ + pushd build ; \ + cmake .. -GNinja -DBUILD_CLIENTS_ONLY=ON -DBUILD_PYTHON=OFF \ + -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + popd ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd + +# Seedee +RUN git clone https://gitlab.desy.de/fs-sc/seedee.git ; \ + pushd seedee ; \ + mkdir build ; \ + cd build ; \ + cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$CFPREFIX -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + cd .. ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd diff --git a/deployment/docker/Dockerfile_centos7 b/deployment/docker/Dockerfile_centos7 new file mode 100644 index 00000000..5e591b04 --- /dev/null +++ b/deployment/docker/Dockerfile_centos7 @@ -0,0 +1,133 @@ +FROM centos:centos7 + +## Following hack is required for Centos7 image as Centos7 is EOL. +## yum-config-manager --nogpgcheck --add-repo https://archive.kernel.org/centos-vault/7.9.2009/os/x86_64/Packages/ +## yum-config-manager --nogpgcheck --add-repo http://archive.kernel.org/centos-vault/7.9.2009/updates/x86_64/ +## yum-config-manager --nogpgcheck --add-repo https://mirror.nsc.liu.se/centos-store/7.9.2009/os/x86_64/Packages/ +## yum-config-manager --nogpgcheck --add-repo https://mirror.nsc.liu.se/centos-store/7.9.2009/updates/x86_64/Packages/ + +ENV CFPREFIX=/software/crystfel/devel +ENV PKG_CONFIG_PATH=$CFPREFIX/lib64/pkgconfig:$CFPREFIX/lib/pkgconfig +ENV LD_LIBRARY_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV LD_RUN_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV PATH=$CFPREFIX/bin:/opt/cmake-3.18.2-Linux-x86_64/bin/:$PATH + +RUN yum-config-manager --disable \* && \ + yum-config-manager --nogpgcheck --add-repo https://vault.centos.org/7.9.2009/os/x86_64/ && \ + yum-config-manager --nogpgcheck --add-repo https://vault.centos.org/7.9.2009/updates/x86_64/ **\ + yum-config-manager --enable vault.centos.org_7.9.2009_os_x86_64_ && \ + yum-config-manager --save --setopt=vault.centos.org_7.9.2009_os_x86_64_.gpgkey="file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7" && \ + yum-config-manager --enable vault.centos.org_7.9.2009_updates_x86_64_ && \ + yum-config-manager --save --setopt=vault.centos.org_7.9.2009_updates_x86_64_.gpgkey="file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7" + +RUN yum -y update ; \ + yum -y group install 'Development Tools' ; \ + yum -y install flex bison gcc-c++ gcc-gfortran gtk3-devel cairo-devel \ + pango-devel gdk-pixbuf2-devel libpng-devel diffutils \ + zeromq-devel python3-devel which wget lz4 lz4-devel \ + bzip2-libs bzip2-devel libcurl-devel openssh-clients krb5-workstation cmake + +RUN sed -i -E '/default_ccache_name/s//# default_ccache_name/g' /etc/krb5.conf ; \ + sed -i -E '/#\s+default_realm\s* =\s*EXAMPLE.COM\s*/s//default_realm = DESY.DE/g' /etc/krb5.conf ; \ + if [ ! -d ~/.ssh ]; then \ + mkdir -p ~/.ssh; \ + chmod 700 ~/.ssh; \ + fi; \ + wget -q https://wims.desy.de/system/ALL_afs/etc/ssh_known_hosts2 -O ~/.ssh/known_hosts + +RUN python3 -m ensurepip --upgrade && python3 -m pip install meson ninja + +# CMake +RUN cd /opt; \ + wget -nv https://github.com/Kitware/CMake/releases/download/v3.18.2/cmake-3.18.2-Linux-x86_64.tar.gz; \ + tar -xzf cmake-3.18.2-Linux-x86_64.tar.gz; \ + rm ./cmake-3.18.2-Linux-x86_64.tar.gz + +# HDF5 +RUN wget -nv https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/src/hdf5-1.12.1.tar.bz2 ; \ + tar -xf hdf5-1.12.1.tar.bz2 ; \ + pushd hdf5-1.12.1; \ + ./configure --prefix=$CFPREFIX \ + --enable-shared --disable-static \ + --with-default-plugindir=${CFPREFIX}/lib64/plugins ;\ + make -j 32; \ + make install; \ + popd; \ + rm -rf hdf5-1.12.1 hdf5-1.12.1.tar.bz2 + +# HDF5 filter plugins +RUN git clone --depth 1 https://github.com/nexusformat/HDF5-External-Filter-Plugins.git ; \ + pushd HDF5-External-Filter-Plugins; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DENABLE_BITSHUFFLE_PLUGIN=yes \ + -DENABLE_LZ4_PLUGIN=yes \ + -DENABLE_BZIP2_PLUGIN=yes \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd; \ + make -C build -j 32; \ + make -C build install; \ + popd ; \ + rm -rf HDF5-External-Filter-Plugins + +# GSL +RUN wget -nv http://ftp.fau.de/gnu/gsl/gsl-2.6.tar.gz ; \ + tar -xzf gsl-2.6.tar.gz ; \ + pushd gsl-2.6 ; \ + ./configure --prefix=$CFPREFIX ; \ + make -j 32 ; \ + make install ; \ + popd ; \ + rm -rf gsl-2.6 gsl-2.6.tar.gz + +# FFTW +RUN wget -nv http://fftw.org/fftw-3.3.9.tar.gz ; \ + tar -xzf fftw-3.3.9.tar.gz ; \ + pushd fftw-3.3.9 ; \ + ./configure --enable-shared --prefix=$CFPREFIX ; \ + make -j 32 ; \ + make install ; \ + popd ; \ + rm -rf fftw-3.3.9 fftw-3.3.9.tar.gz ; + +# MsgPack +RUN wget -nv https://github.com/msgpack/msgpack-c/releases/download/cpp-3.3.0/msgpack-3.3.0.tar.gz ; \ + tar -xzf msgpack-3.3.0.tar.gz ; \ + pushd msgpack-3.3.0; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DMSGPACK_BUILD_EXAMPLES=OFF \ + -DMSGPACK_ENABLE_CXX=OFF \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd ; \ + make -C build -j 32 ; \ + make -C build install ; \ + popd ; \ + rm -rf msgpack-3.3.0 msgpack-3.3.0.tar.gz + +# ASAP::O +RUN git clone https://gitlab.desy.de/asapo/asapo.git ; \ + pushd asapo ; \ + git checkout 23.12.0-alpha ; \ + mkdir build ; \ + pushd build ; \ + cmake .. -GNinja -DBUILD_CLIENTS_ONLY=ON -DBUILD_PYTHON=OFF \ + -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + popd ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd + +# Seedee +RUN git clone https://gitlab.desy.de/fs-sc/seedee.git ; \ + pushd seedee ; \ + mkdir build ; \ + cd build ; \ + cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$CFPREFIX -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + cd .. ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd diff --git a/deployment/docker/Dockerfile_fedora b/deployment/docker/Dockerfile_fedora new file mode 100644 index 00000000..52b9fb03 --- /dev/null +++ b/deployment/docker/Dockerfile_fedora @@ -0,0 +1,79 @@ +FROM fedora:latest + +ENV CFPREFIX=/software/crystfel/devel +ENV PKG_CONFIG_PATH=$CFPREFIX/lib64/pkgconfig:$CFPREFIX/lib/pkgconfig +ENV LD_LIBRARY_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV LD_RUN_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV PATH=$CFPREFIX/bin:$PATH + +RUN dnf -y install epel-release ; \ + dnf -y update ; \ + dnf -y group install development-tools; \ + dnf -y install gsl-devel flex bison gcc-c++ gtk3-devel cairo-devel \ + pango-devel gdk-pixbuf2-devel fftw-devel libpng-devel diffutils \ + hdf5-devel zeromq-devel python3-devel which wget lz4 lz4-devel \ + bzip2-libs bzip2-devel libcurl-devel openssh-clients krb5-workstation cmake \ + gcc-gfortran +RUN sed -i -E '/default_ccache_name/s//# default_ccache_name/g' /etc/krb5.conf ; \ + sed -i -E '/#\s+default_realm\s* =\s*EXAMPLE.COM\s*/s//default_realm = DESY.DE/g' /etc/krb5.conf ; \ + if [ ! -d ~/.ssh ]; then \ + mkdir -p ~/.ssh; \ + chmod 700 ~/.ssh; \ + fi; \ + wget -q https://wims.desy.de/system/ALL_afs/etc/ssh_known_hosts2 -O ~/.ssh/known_hosts +RUN python3 -m ensurepip --upgrade && python3 -m pip install meson ninja + +# HDF5 filter plugins +RUN git clone --depth 1 https://github.com/nexusformat/HDF5-External-Filter-Plugins.git ; \ + pushd HDF5-External-Filter-Plugins; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DENABLE_BITSHUFFLE_PLUGIN=yes \ + -DENABLE_LZ4_PLUGIN=yes \ + -DENABLE_BZIP2_PLUGIN=yes \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd; \ + make -C build -j 32; \ + make -C build install; \ + popd # HDF5-External-Filter-Plugins + +# MsgPack +RUN wget -nv https://github.com/msgpack/msgpack-c/releases/download/cpp-3.3.0/msgpack-3.3.0.tar.gz ; \ + tar -xzf msgpack-3.3.0.tar.gz ; \ + pushd msgpack-3.3.0; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DMSGPACK_BUILD_EXAMPLES=OFF \ + -DMSGPACK_ENABLE_CXX=OFF \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd ; \ + make -C build -j 32 ; \ + make -C build install ; \ + popd # msgpack-3.3.0 + +# ASAP::O +RUN git clone https://gitlab.desy.de/asapo/asapo.git ; \ + pushd asapo ; \ + git checkout 23.12.0-alpha ; \ + mkdir build ; \ + pushd build ; \ + cmake .. -GNinja -DBUILD_CLIENTS_ONLY=ON -DBUILD_PYTHON=OFF \ + -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + popd ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd + +# Seedee +RUN git clone https://gitlab.desy.de/fs-sc/seedee.git ; \ + pushd seedee ; \ + mkdir build ; \ + cd build ; \ + cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$CFPREFIX -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + cd .. ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd diff --git a/deployment/docker/Dockerfile_ubuntu b/deployment/docker/Dockerfile_ubuntu new file mode 100644 index 00000000..db2eafa3 --- /dev/null +++ b/deployment/docker/Dockerfile_ubuntu @@ -0,0 +1,80 @@ +FROM ubuntu:latest + +SHELL ["/bin/bash", "-c"] +ENV CFPREFIX=/software/crystfel/devel +ENV PKG_CONFIG_PATH=$CFPREFIX/lib64/pkgconfig:$CFPREFIX/lib/pkgconfig +ENV LD_LIBRARY_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV LD_RUN_PATH=$CFPREFIX/lib64:$CFPREFIX/lib +ENV PATH=$CFPREFIX/bin:/venv/bin:$PATH + +RUN export DEBIAN_FRONTEND=noninteractive; \ + apt-get -y update ; \ + apt-get -q -y install build-essential flex bison cmake gfortran libgsl-dev libgtk-3-dev \ + libcairo2-dev libpango1.0-dev libgdk-pixbuf2.0-dev libfftw3-dev libpng-dev libmsgpack-dev \ + libeigen3-dev libccp4-dev libhdf5-dev libzmq3-dev libpython3-dev python3-full python3-venv \ + liblz4-dev libbz2-dev libcurl4-openssl-dev debianutils wget lz4 diffutils openssh-client \ + krb5-user sssd-krb5 git curl unzip + +RUN sed -i -E '/default_ccache_name/s//# default_ccache_name/g' /etc/krb5.conf ; \ + sed -i -E '/#\s+default_realm\s* =\s*EXAMPLE.COM\s*/s//default_realm = DESY.DE/g' /etc/krb5.conf ; \ + if [ ! -d ~/.ssh ]; then \ + mkdir -p ~/.ssh; \ + chmod 700 ~/.ssh; \ + fi; \ + wget -q https://wims.desy.de/system/ALL_afs/etc/ssh_known_hosts2 -O ~/.ssh/known_hosts +RUN python3 -m venv ./venv; /venv/bin/pip3 install meson ninja + +# HDF5 filter plugins +RUN git clone --depth 1 https://github.com/nexusformat/HDF5-External-Filter-Plugins.git ; \ + pushd HDF5-External-Filter-Plugins; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DENABLE_BITSHUFFLE_PLUGIN=yes \ + -DENABLE_LZ4_PLUGIN=yes \ + -DENABLE_BZIP2_PLUGIN=yes \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd; \ + make -C build -j 32; \ + make -C build install; \ + popd # HDF5-External-Filter-Plugins + +# MsgPack +RUN wget -nv https://github.com/msgpack/msgpack-c/releases/download/cpp-3.3.0/msgpack-3.3.0.tar.gz ; \ + tar -xzf msgpack-3.3.0.tar.gz ; \ + pushd msgpack-3.3.0; \ + mkdir build; \ + pushd build; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DMSGPACK_BUILD_EXAMPLES=OFF \ + -DMSGPACK_ENABLE_CXX=OFF \ + -DCMAKE_BUILD_TYPE=Release ; \ + popd ; \ + make -C build -j 32 ; \ + make -C build install ; \ + popd # msgpack-3.3.0 + +# ASAP::O +RUN git clone https://gitlab.desy.de/asapo/asapo.git ; \ + pushd asapo ; \ + git checkout 23.12.0-alpha ; \ + mkdir build ; \ + pushd build ; \ + cmake .. -GNinja -DBUILD_CLIENTS_ONLY=ON -DBUILD_PYTHON=OFF \ + -DCMAKE_INSTALL_PREFIX=$CFPREFIX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + popd ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd + +# Seedee +RUN git clone https://gitlab.desy.de/fs-sc/seedee.git ; \ + pushd seedee ; \ + mkdir build ; \ + cd build ; \ + cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$CFPREFIX -DCMAKE_BUILD_TYPE=RelWithDebInfo ; \ + cd .. ; \ + ninja -C build ; \ + ninja -C build install ; \ + popd diff --git a/deployment/gitlab/config.toml b/deployment/gitlab/config.toml new file mode 100644 index 00000000..92e1ad3e --- /dev/null +++ b/deployment/gitlab/config.toml @@ -0,0 +1,16 @@ +[[runners]] + name = "macos_runner.desy.de" + url = "https://gitlab.desy.de/" + id = 9999 + token = "registration token" + token_obtained_at = 2023-05-10T11:19:56Z + token_expires_at = 0001-01-01T00:00:00Z + executor = "shell" + [runners.custom_build_dir] + [runners.cache] + MaxUploadedArchiveSize = 0 + [runners.cache.s3] + [runners.cache.gcs] + [runners.cache.azure] + [runners.feature_flags] + FF_ENABLE_JOB_CLEANUP = true |