ci: container improvements

.gitlab-ci.yml

@@ -40,26 +40,11 @@ builder-image:
   needs:
     - builder-image
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
-  variables:
-    SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "15.0"
-    XCODE_BUILD_ID: 15A240d
   before_script:
-    - echo HOST=$HOST
     - |
-      if [ "$HOST" = "x86_64-apple-darwin" ]; then
-        mkdir -p depends/SDKs
-        mkdir -p depends/sdk-sources
-        OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-        OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
-        if [ ! -f "$OSX_SDK_PATH" ]; then
-          echo "Downloading MacOS SDK"
-          curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
-        fi
-        if [ -f "$OSX_SDK_PATH" ]; then
-          echo "Extracting MacOS SDK"
-          tar -C depends/SDKs -xf "$OSX_SDK_PATH"
-        fi
+      echo HOST=${HOST}
+      if [[ "${HOST}" == "x86_64-apple-darwin" ]]; then
+        ./contrib/containers/guix/scripts/setup-sdk
       fi
   script:
     - make -j$(nproc) -C depends HOST=$HOST $DEP_OPTS

ci/dash/build_depends.sh

@@ -20,17 +20,8 @@ mkdir -p $CACHE_DIR/sdk-sources
 ln -s $CACHE_DIR/depends ${DEPENDS_DIR}/built
 ln -s $CACHE_DIR/sdk-sources ${DEPENDS_DIR}/sdk-sources
 
-mkdir -p ${DEPENDS_DIR}/SDKs
-
-if [ -n "$XCODE_VERSION" ]; then
-  OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-  OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
-  if [ ! -f "$OSX_SDK_PATH" ]; then
-    curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
-  fi
-  if [ -f "$OSX_SDK_PATH" ]; then
-    tar -C ${DEPENDS_DIR}/SDKs -xf "$OSX_SDK_PATH"
-  fi
+if [[ "${HOST}" == "x86_64-apple-darwin" ]]; then
+    ./contrib/containers/guix/scripts/setup-sdk
 fi
 
 make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS

contrib/containers/ci/Dockerfile

@@ -7,14 +7,17 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London"
 # (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries)
 ENV APT_ARGS="-y --no-install-recommends --no-upgrade"
 
-
-# Install packages for i386; disabled on aarch64 and arm64 hosts
-RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || dpkg --add-architecture i386)
-RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || (apt-get update && apt-get install $APT_ARGS \
-    g++-multilib \
-    wine32) && rm -rf /var/lib/apt/lists/*)
-
-RUN apt-get update && apt-get install $APT_ARGS \
+# Install packages for i386 on amd64 hosts, then install common packages
+RUN set -ex; \
+    apt-get update && \
+    if [ "$(dpkg --print-architecture)" = "amd64" ]; then \
+        dpkg --add-architecture i386 && \
+        apt-get update && \
+        apt-get install $APT_ARGS \
+        g++-multilib \
+        wine32; \
+    fi; \
+    apt-get install $APT_ARGS \
     autotools-dev \
     automake \
     autoconf \
@@ -23,13 +26,11 @@ RUN apt-get update && apt-get install $APT_ARGS \
     bsdmainutils \
     curl \
     ccache \
-    clang \
     cmake \
     g++ \
     gettext \
     git \
-    libc++-dev \
-    libc++abi-dev \
+    gnupg \
     libtool \
     libxcb-icccm4 \
     libxcb-image0 \
@@ -42,11 +43,38 @@ RUN apt-get update && apt-get install $APT_ARGS \
     libxcb-xinerama0 \
     libxcb-xkb1 \
     libxkbcommon-x11-0 \
-    wget \
+    lsb-release \
+    software-properties-common \
     unzip \
+    wget \
     m4 \
     pkg-config \
-    zlib1g-dev
+    zlib1g-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Clang+LLVM and set it as default
+# We don't need all packages but the default set doesn't include some
+# packages we want so we will need to install some of them manually.
+ARG LLVM_VERSION=16
+RUN set -ex; \
+    echo "Installing LLVM and Clang ${LLVM_VERSION}..."; \
+    curl -sL https://apt.llvm.org/llvm.sh | bash -s "${LLVM_VERSION}"; \
+    echo "Installing additional packages..."; \
+    apt-get update && apt-get install $APT_ARGS \
+    "clang-format-${LLVM_VERSION}" \
+    "clang-tidy-${LLVM_VERSION}" \
+    "libc++-${LLVM_VERSION}-dev" \
+    "libc++abi-${LLVM_VERSION}-dev" \
+    "libclang-rt-${LLVM_VERSION}-dev"; \
+    rm -rf /var/lib/apt/lists/*; \
+    echo "Setting defaults..."; \
+    lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \
+    for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \
+        lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \
+    done; \
+    sh -c "${lldbUpdAltArgs}";
+# LD_LIBRARY_PATH is empty by default, this is the first entry
+ENV LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib"
 
 # Python setup
 # PYTHON_VERSION should match the value in .python-version
@@ -61,10 +89,11 @@ RUN apt-get update && apt-get install $APT_ARGS \
     libreadline-dev \
     libsqlite3-dev \
     libssl-dev \
-    llvm \
     make \
     tk-dev \
-    xz-utils
+    xz-utils \
+    && rm -rf /var/lib/apt/lists/*
+
 ENV PYENV_ROOT="/usr/local/pyenv"
 ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"
 RUN curl https://pyenv.run | bash \
@@ -82,19 +111,38 @@ RUN pip3 install \
     pyzmq==22.3.0 \
     vulture==2.3
 
-# dash_hash
 ARG DASH_HASH_VERSION=1.4.0
-RUN git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash
-RUN cd dash_hash && pip3 install -r requirements.txt .
+RUN set -ex; \
+    cd /tmp; \
+    git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash; \
+    cd dash_hash && pip3 install -r requirements.txt .; \
+    cd .. && rm -rf dash_hash
+
+ARG CPPCHECK_VERSION=2.13.0
+RUN set -ex; \
+    curl -fL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" -o /tmp/cppcheck.tar.gz; \
+    mkdir -p /opt/cppcheck && tar -xzf /tmp/cppcheck.tar.gz -C /opt/cppcheck --strip-components=1 && rm /tmp/cppcheck.tar.gz; \
+    cd /opt/cppcheck; \
+    mkdir build && cd build && cmake .. && cmake --build . -j "$(( $(nproc) - 1 ))"; \
+    mkdir /usr/local/share/Cppcheck && ln -s /opt/cppcheck/cfg/ /usr/local/share/Cppcheck/cfg; \
+    rm -rf /tmp/cppcheck.tar.gz
+ENV PATH="/opt/cppcheck/build/bin:${PATH}"
 
+ARG SHELLCHECK_VERSION=v0.7.1
+RUN set -ex; \
+    curl -fL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" -o /tmp/shellcheck.tar.xz; \
+    mkdir -p /opt/shellcheck && tar -xf /tmp/shellcheck.tar.xz -C /opt/shellcheck --strip-components=1 && rm /tmp/shellcheck.tar.xz
+ENV PATH="/opt/shellcheck:${PATH}"
+
+# Add user with specified (or default) user/group ids and setup configuration files
 ARG USER_ID=1000
 ARG GROUP_ID=1000
-
-# add user with specified (or default) user/group ids
-ENV USER_ID="${USER_ID}"
-ENV GROUP_ID="${GROUP_ID}"
-RUN groupadd -g ${GROUP_ID} dash
-RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash
+RUN set -ex; \
+    groupadd -g ${GROUP_ID} dash; \
+    useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash; \
+    mkdir -p /home/dash/.config/gdb; \
+    echo "add-auto-load-safe-path /usr/lib/llvm-${LLVM_VERSION}/lib" | tee /home/dash/.config/gdb/gdbinit; \
+    chown ${USER_ID}:${GROUP_ID} -R /home/dash
 
 # Packages needed for all target builds
 RUN apt-get update && apt-get install $APT_ARGS \
@@ -111,17 +159,8 @@ RUN apt-get update && apt-get install $APT_ARGS \
     valgrind \
     wine-stable \
     wine64 \
-    xorriso
-
-ARG CPPCHECK_VERSION=2.13.0
-RUN curl -sL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" | tar -xvzf - --directory /tmp/
-RUN cd /tmp/cppcheck-${CPPCHECK_VERSION} && mkdir build && cd build && cmake .. && cmake --build . -j 8
-ENV PATH="/tmp/cppcheck-${CPPCHECK_VERSION}/build/bin:${PATH}"
-RUN mkdir /usr/local/share/Cppcheck && ln -s /tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ /usr/local/share/Cppcheck/cfg
-
-ARG SHELLCHECK_VERSION=v0.7.1
-RUN curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/
-ENV PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}"
+    xorriso \
+    && rm -rf /var/lib/apt/lists/*
 
 # This is a hack. It is needed because gcc-multilib and g++-multilib are conflicting with g++-arm-linux-gnueabihf. This is
 # due to gcc-multilib installing the following symbolic link, which is needed for -m32 support. However, this causes
@@ -135,20 +174,6 @@ RUN \
   update-alternatives --set x86_64-w64-mingw32-g++  /usr/bin/x86_64-w64-mingw32-g++-posix; \
   exit 0
 
-ARG LLVM_VERSION=16
-# Setup Clang+LLVM support
-RUN apt-get update && apt-get install $APT_ARGS \
-    lsb-release \
-    software-properties-common \
-    gnupg \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN cd /tmp && \
-    wget https://apt.llvm.org/llvm.sh && \
-    chmod +x llvm.sh && \
-    /tmp/llvm.sh ${LLVM_VERSION} && \
-    rm -rf /tmp/llvm.sh
-
 RUN \
   mkdir -p /src/dash && \
   mkdir -p /cache/ccache && \

contrib/containers/develop/docker-compose.yml

@@ -1,17 +1,18 @@
-version: "3.9"
 services:
   container:
     entrypoint: /bin/bash
     build:
       context: '..'
       dockerfile: './develop/Dockerfile'
-    tty: true         # Equivalent to -t
-    stdin_open: true  # Equivalent to -i
     ports:
       - "9998:9998"           # Mainnet Ports
       - "9999:9999"
       - "19998:19998"         # Testnet Ports
       - "19999:19999"
+    security_opt:
+      - seccomp:unconfined
+    stdin_open: true  # Equivalent to -i
+    tty: true         # Equivalent to -t
 
 # A note about volumes:
 #

contrib/containers/guix/Dockerfile

@@ -18,7 +18,11 @@ RUN apt-get update && \
     sudo \
     wget \
     xz-utils && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/*; \
+    targetLocale="en_US.UTF-8"; \
+    locale-gen ${targetLocale} && \
+    update-locale LC_ALL=${targetLocale} && \
+    update-locale LANG=${targetLocale};
 
 ARG guix_download_path=ftp://ftp.gnu.org/gnu/guix
 ARG guix_version=1.4.0
@@ -30,8 +34,7 @@ ENV PATH="/usr/local/bin:/usr/local/guix/current/bin:$PATH"
 
 # Application Setup
 # https://guix.gnu.org/manual/en/html_node/Application-Setup.html
-ENV GUIX_LOCPATH="/usr/local/guix/profile" \
-    LC_ALL="en_US.UTF-8"
+ENV GUIX_LOCPATH="/usr/local/guix/profile"
 
 RUN guix_file_name=guix-binary-${guix_version}.$(uname -m)-linux.tar.xz          && \
     eval "guix_checksum=\${guix_checksum_$(uname -m)}"                           && \
@@ -74,18 +77,19 @@ RUN usermod -aG sudo ${USERNAME} && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
 # Copy required files to container
-COPY --from=docker_root ./motd.txt           /etc/motd
-COPY --from=docker_root ./scripts/entrypoint /usr/local/bin/entrypoint
-COPY --from=docker_root ./scripts/guix-check /usr/local/bin/guix-check
-COPY --from=docker_root ./scripts/guix-start /usr/local/bin/guix-start
+COPY --from=docker_root ./motd.txt             /etc/motd
+COPY --from=docker_root ./scripts/entrypoint   /usr/local/bin/entrypoint
+COPY --from=docker_root ./scripts/guix-check   /usr/local/bin/guix-check
+COPY --from=docker_root ./scripts/guix-start   /usr/local/bin/guix-start
+COPY --from=docker_root ./scripts/setup-sdk    /usr/local/bin/setup-sdk
 
 # Create directories for mounting to save/restore cache and grant necessary permissions
 RUN mkdir -p \
         /home/${USERNAME}/.cache \
-        /src/dash/depends/{built,sources,work} && \
+        /src/dash/depends/{built,sources,work}; \
     chown -R ${USER_ID}:${GROUP_ID} \
         /home/${USERNAME}/.cache \
-        /src
+        /src;
 
 WORKDIR "/src/dash"
 

contrib/containers/guix/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.9"
 services:
   guix_ubuntu:
     build:

contrib/containers/guix/scripts/guix-start

@@ -9,19 +9,10 @@ if [[ ! -d "${WORKSPACE_PATH}" || ! "${WORKSPACE_PATH}" = /* || ! -f "${WORKSPAC
     exit 1
 fi
 
-XCODE_VERSION="15.0"
-XCODE_RELEASE="15A240d"
-XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
-XCODE_SOURCE="${XCODE_SOURCE:-https://bitcoincore.org/depends-sources/sdks}"
-
 export SDK_PATH="${SDK_PATH:-${WORKSPACE_PATH}/depends/SDKs}"
+export SDK_SRCS="${SDK_PATH:-${WORKSPACE_PATH}/depends/sdk-sources}"
 
-# Check if macOS SDK is present, if not, download it
-if [[ ! -d "${SDK_PATH}/${XCODE_ARCHIVE}" ]]; then
-    echo "Preparing macOS SDK..."
-    mkdir -p "${SDK_PATH}"
-    curl -L "${XCODE_SOURCE}/${XCODE_ARCHIVE}.tar.gz" | tar -xz -C "${SDK_PATH}"
-fi
+./contrib/containers/guix/scripts/setup-sdk
 
 # Add safe.directory option only when WORKSPACE_PATH was specified via cmd-line arguments (happens in CI)
 if [[ -n "${1}" ]]; then

contrib/containers/guix/scripts/setup-sdk

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright (c) 2024 The Dash Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+set -eo pipefail
+
+SDK_URL="${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}"
+SDK_PATH="${SDK_PATH:-depends/SDKs}"
+SDK_SRCS="${SDK_SOURCES:-depends/sdk-sources}"
+XCODE_VERSION="${XCODE_VERSION:-15.0}"
+XCODE_RELEASE="${XCODE_RELEASE:-15A240d}"
+XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
+XCODE_AR_PATH="${SDK_SRCS}/${XCODE_ARCHIVE}.tar.gz"
+
+if [ ! -d "${SDK_PATH}/${XCODE_ARCHIVE}" ]; then
+    if [ ! -f "${XCODE_AR_PATH}" ]; then
+        echo "Downloading macOS SDK..."
+        mkdir -p "${SDK_SRCS}"
+        curl --location --fail "${SDK_URL}/${XCODE_ARCHIVE}.tar.gz" -o "${XCODE_AR_PATH}"
+    fi
+    echo "Extracting macOS SDK..."
+    mkdir -p "${SDK_PATH}"
+    tar -C "${SDK_PATH}" -xf "${XCODE_AR_PATH}"
+fi