From 95228016751d14c32d3f1a219408cc32e8478d62 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:52:43 +0000 Subject: [PATCH 01/11] [don't mind me, this'll go away after we rebase] --- .github/workflows/build.yml | 79 +++++++++---------- .github/workflows/matrix.json | 66 ++++++++++++++++ .gitlab-ci.yml | 28 +++---- ci/dash/matrix.sh | 22 +++--- ci/dash/test_integrationtests.sh | 2 +- ci/dash/test_unittests.sh | 4 +- ci/test/00_setup_env.sh | 1 - ci/test/00_setup_env_native_fuzz.sh | 2 +- .../00_setup_env_native_fuzz_with_valgrind.sh | 2 +- ci/test/00_setup_env_native_multiprocess.sh | 4 +- ci/test/00_setup_env_native_tsan.sh | 6 +- ci/test/00_setup_env_native_ubsan.sh | 4 +- ci/test/00_setup_env_native_valgrind.sh | 2 +- ci/test/00_setup_env_s390x.sh | 1 - contrib/containers/ci/Dockerfile | 48 ++++++----- contrib/containers/develop/docker-compose.yml | 7 +- depends/packages/qt.mk | 12 ++- 17 files changed, 184 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/matrix.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7b9dbd80207f2..7e39b112306f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,8 @@ jobs: outputs: image-tag: ${{ steps.prepare.outputs.image-tag }} repo-name: ${{ steps.prepare.outputs.repo-name }} + dep-matrix: ${{ steps.prepare.outputs.dep-matrix }} + src-matrix: ${{ steps.prepare.outputs.src-matrix }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -32,6 +34,8 @@ jobs: REPO_NAME=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') echo "image-tag=${BRANCH_NAME}" >> $GITHUB_OUTPUT echo "repo-name=${REPO_NAME}" >> $GITHUB_OUTPUT + echo "dep-matrix=$(jq -r '.dep' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT + echo "src-matrix=$(jq -r '.src' -c .github/workflows/matrix.json)" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -56,18 +60,12 @@ jobs: cache-to: type=inline build-depends: - name: Build Dependencies + name: ${{ matrix.depends_name }} needs: build-image runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - include: - - build_target: arm-linux - host: arm-linux-gnueabihf - - build_target: linux64 - host: x86_64-pc-linux-gnu - + matrix: ${{ fromJson(needs.build-image.outputs.dep-matrix) }} container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -86,52 +84,36 @@ jobs: restore-keys: | depends-sources- + - name: Determine params + id: det-params + run: | + dep_name="${{ matrix.depends_name }}" + dep_opts="${{ matrix.dep_opts }}" + dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" + echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT + - name: Cache depends uses: actions/cache@v4 with: path: | depends/built depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} restore-keys: | - ${{ runner.os }}-depends-${{ matrix.build_target }}-${{ hashFiles('depends/packages/*') }} - ${{ runner.os }}-depends-${{ matrix.build_target }} + ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- + ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Build depends - run: make -j$(nproc) -C depends HOST=${{ matrix.host }} + run: env HOST=${{ matrix.host }} ${{ matrix.dep_opts }} make -j$(nproc) -C depends build: - name: Build + name: ${{ matrix.build_target }} needs: [build-image, build-depends] runs-on: ubuntu-22.04 strategy: fail-fast: false - matrix: - include: - - build_target: arm-linux - host: arm-linux-gnueabihf - depends_on: arm-linux - - build_target: linux64 - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_tsan - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_ubsan - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_fuzz - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_cxx20 - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_sqlite - host: x86_64-pc-linux-gnu - depends_on: linux64 - - build_target: linux64_nowallet - host: x86_64-pc-linux-gnu - depends_on: linux64 + matrix: ${{ fromJson(needs.build-image.outputs.src-matrix) }} container: image: ghcr.io/${{ needs.build-image.outputs.repo-name }}/dashcore-ci-runner:${{ needs.build-image.outputs.image-tag }} options: --user root @@ -141,13 +123,28 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} + - name: Determine params + id: det-params + run: | + dep_name="${{ matrix.depends_name }}" + dep_opts="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .dep_opts" -c .github/workflows/matrix.json)" + dep_hash="$(echo -n ${dep_opts} | sha256sum | head -c 64)" + echo "\"${dep_name}\" has DEP_OPTS \"${dep_opts}\" with hash \"${dep_hash}\"" + echo "dep_hash=${dep_hash}" >> $GITHUB_OUTPUT + dep_host="$(jq -r ".dep.include[] | select(.depends_name == \"${{ matrix.depends_name }}\") | .host" -c .github/workflows/matrix.json)" + echo "\"${dep_name}\" has HOST \"${dep_host}\"" + echo "dep_host=${dep_host}" >> $GITHUB_OUTPUT + - name: Restore depends cache uses: actions/cache/restore@v4 with: path: | depends/built - depends/${{ matrix.host }} - key: ${{ runner.os }}-depends-${{ matrix.depends_on }}-${{ hashFiles('depends/packages/*') }} + depends/${{ steps.det-params.outputs.dep_host }} + key: ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}-${{ steps.det-params.outputs.dep_hash }} + restore-keys: | + ${{ runner.os }}-depends-${{ matrix.depends_name }}-${{ hashFiles('depends/packages/*') }}- + ${{ runner.os }}-depends-${{ matrix.depends_name }}- - name: Determine PR Base SHA id: vars diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json new file mode 100644 index 0000000000000..dd3cd1b4d988f --- /dev/null +++ b/.github/workflows/matrix.json @@ -0,0 +1,66 @@ +{ + "dep": { + "include": [ + { + "depends_name": "arm-linux-gnueabihf", + "host": "arm-linux-gnueabihf", + "dep_opts": "" + }, + { + "depends_name": "x86_64-pc-linux-gnu_debug", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "DEBUG=1" + }, + { + "depends_name": "x86_64-pc-linux-gnu_multiprocess", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" + }, + { + "depends_name": "x86_64-pc-linux-gnu_nowallet", + "host": "x86_64-pc-linux-gnu", + "dep_opts": "NO_WALLET=1" + } + ] + }, + "src": { + "include": [ + { + "build_target": "arm-linux", + "depends_name": "arm-linux-gnueabihf" + }, + { + "build_target": "linux64", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_cxx20", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_fuzz", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_multiprocess", + "depends_name": "x86_64-pc-linux-gnu_multiprocess" + }, + { + "build_target": "linux64_nowallet", + "depends_name": "x86_64-pc-linux-gnu_nowallet" + }, + { + "build_target": "linux64_sqlite", + "depends_name": "x86_64-pc-linux-gnu_debug" + }, + { + "build_target": "linux64_tsan", + "depends_name": "x86_64-pc-linux-gnu_multiprocess" + }, + { + "build_target": "linux64_ubsan", + "depends_name": "x86_64-pc-linux-gnu_debug" + } + ] + } +} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6af1abbbe6382..477d1f01a84dc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -189,13 +189,13 @@ x86_64-w64-mingw32: variables: HOST: x86_64-w64-mingw32 -x86_64-pc-linux-gnu-debug: +x86_64-pc-linux-gnu_debug: extends: .build-depends-template variables: HOST: x86_64-pc-linux-gnu DEP_OPTS: "DEBUG=1" -x86_64-pc-linux-gnu-nowallet: +x86_64-pc-linux-gnu_nowallet: extends: - .build-depends-template - .skip-in-fast-mode-template @@ -203,13 +203,13 @@ x86_64-pc-linux-gnu-nowallet: HOST: x86_64-pc-linux-gnu DEP_OPTS: "NO_WALLET=1" -x86_64-pc-linux-gnu-multiprocess: +x86_64-pc-linux-gnu_multiprocess: extends: - .build-depends-template - .skip-in-fast-mode-template variables: HOST: x86_64-pc-linux-gnu - DEP_OPTS: "MULTIPROCESS=1" + DEP_OPTS: "DEBUG=1 MULTIPROCESS=1 CC=clang-18 CXX=clang++-18" x86_64-apple-darwin: extends: @@ -239,7 +239,7 @@ win64-build: linux64-build: extends: .build-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64 @@ -248,7 +248,7 @@ linux64_cxx20-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_cxx20 @@ -257,7 +257,7 @@ linux64_sqlite-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_sqlite @@ -266,7 +266,7 @@ linux64_fuzz-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_fuzz @@ -275,7 +275,7 @@ linux64_fuzz-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu-debug +# - x86_64-pc-linux-gnu_debug # variables: # BUILD_TARGET: linux64_asan @@ -284,7 +284,7 @@ linux64_tsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_multiprocess variables: BUILD_TARGET: linux64_tsan @@ -293,7 +293,7 @@ linux64_ubsan-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-debug + - x86_64-pc-linux-gnu_debug variables: BUILD_TARGET: linux64_ubsan @@ -302,7 +302,7 @@ linux64_nowallet-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-nowallet + - x86_64-pc-linux-gnu_nowallet variables: BUILD_TARGET: linux64_nowallet @@ -311,7 +311,7 @@ linux64_multiprocess-build: - .build-template - .skip-in-fast-mode-template needs: - - x86_64-pc-linux-gnu-multiprocess + - x86_64-pc-linux-gnu_multiprocess variables: BUILD_TARGET: linux64_multiprocess @@ -320,7 +320,7 @@ linux64_multiprocess-build: # - .build-template # - .skip-in-fast-mode-template # needs: -# - x86_64-pc-linux-gnu-debug +# - x86_64-pc-linux-gnu_debug # variables: # BUILD_TARGET: linux64_valgrind diff --git a/ci/dash/matrix.sh b/ci/dash/matrix.sh index 82a9233df17b9..9822a727890b5 100755 --- a/ci/dash/matrix.sh +++ b/ci/dash/matrix.sh @@ -18,28 +18,30 @@ export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ if [ "$BUILD_TARGET" = "arm-linux" ]; then source ./ci/test/00_setup_env_arm.sh -elif [ "$BUILD_TARGET" = "win64" ]; then - source ./ci/test/00_setup_env_win64.sh elif [ "$BUILD_TARGET" = "linux64" ]; then source ./ci/test/00_setup_env_native_qt5.sh elif [ "$BUILD_TARGET" = "linux64_asan" ]; then source ./ci/test/00_setup_env_native_asan.sh -elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then - source ./ci/test/00_setup_env_native_tsan.sh -elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then - source ./ci/test/00_setup_env_native_ubsan.sh -elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then - source ./ci/test/00_setup_env_native_fuzz.sh elif [ "$BUILD_TARGET" = "linux64_cxx20" ]; then source ./ci/test/00_setup_env_native_cxx20.sh -elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then - source ./ci/test/00_setup_env_native_sqlite.sh +elif [ "$BUILD_TARGET" = "linux64_fuzz" ]; then + source ./ci/test/00_setup_env_native_fuzz.sh +elif [ "$BUILD_TARGET" = "linux64_multiprocess" ]; then + source ./ci/test/00_setup_env_native_multiprocess.sh elif [ "$BUILD_TARGET" = "linux64_nowallet" ]; then source ./ci/test/00_setup_env_native_nowallet.sh +elif [ "$BUILD_TARGET" = "linux64_sqlite" ]; then + source ./ci/test/00_setup_env_native_sqlite.sh +elif [ "$BUILD_TARGET" = "linux64_tsan" ]; then + source ./ci/test/00_setup_env_native_tsan.sh +elif [ "$BUILD_TARGET" = "linux64_ubsan" ]; then + source ./ci/test/00_setup_env_native_ubsan.sh elif [ "$BUILD_TARGET" = "linux64_valgrind" ]; then source ./ci/test/00_setup_env_native_valgrind.sh elif [ "$BUILD_TARGET" = "mac" ]; then source ./ci/test/00_setup_env_mac.sh elif [ "$BUILD_TARGET" = "s390x" ]; then source ./ci/test/00_setup_env_s390x.sh +elif [ "$BUILD_TARGET" = "win64" ]; then + source ./ci/test/00_setup_env_win64.sh fi diff --git a/ci/dash/test_integrationtests.sh b/ci/dash/test_integrationtests.sh index 8a0013b1a94d8..86a00c8f1b0ce 100755 --- a/ci/dash/test_integrationtests.sh +++ b/ci/dash/test_integrationtests.sh @@ -42,7 +42,7 @@ echo "Using socketevents mode: $SOCKETEVENTS" EXTRA_ARGS="--dashd-arg=-socketevents=$SOCKETEVENTS" set +e -LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS +LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ./test/functional/test_runner.py --ci --attempts=3 --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --failfast --nocleanup --tmpdir=$(pwd)/testdatadirs $PASS_ARGS $EXTRA_ARGS RESULT=$? set -e diff --git a/ci/dash/test_unittests.sh b/ci/dash/test_unittests.sh index b3212327107fd..37a0a4575e079 100755 --- a/ci/dash/test_unittests.sh +++ b/ci/dash/test_unittests.sh @@ -29,8 +29,8 @@ if [ "$DIRECT_WINE_EXEC_TESTS" = "true" ]; then wine ./src/test/test_dash.exe else if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - ${TEST_RUNNER_ENV} ./src/test/test_dash --catch_system_errors=no -l test_suite + ./src/test/test_dash --catch_system_errors=no -l test_suite else - ${TEST_RUNNER_ENV} make $MAKEJOBS check VERBOSE=1 + make $MAKEJOBS check VERBOSE=1 fi fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 587dfc2bfefa6..2023705cf8163 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -44,7 +44,6 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4} -export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 2ba78325ff16c..ad69665b6aec7 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -16,4 +16,4 @@ export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16 --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index b5c6b60b4a73a..9860123546744 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -14,4 +14,4 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index c070da4ef0786..b63f7aeb6f79a 100755 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -10,5 +10,5 @@ export CONTAINER_NAME=ci_native_multiprocess export PACKAGES="cmake python3 llvm clang" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang CXX=clang++" # Use clang to avoid OOM -export TEST_RUNNER_ENV="BITCOIND=dash-node" +export BITCOIN_CONFIG="--with-boost-process --enable-debug CC=clang-18 CXX=clang++-18" # Use clang to avoid OOM +export BITCOIND=dash-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 633304580f40e..ffa3d8365c858 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -7,11 +7,11 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libc++abi-16-dev libc++-16-dev python3-zmq" -export DEP_OPTS="CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export PACKAGES="clang-18 llvm-18 libclang-rt-18-dev libc++abi-18-dev libc++-18-dev python3-zmq" +export DEP_OPTS="CC=clang-18 CXX='clang++-18 -stdlib=libc++'" export TEST_RUNNER_EXTRA="--extended --exclude feature_pruning,feature_dbcrash,wallet_multiwallet.py" # Temporarily suppress ASan heap-use-after-free (see issue #14163) export TEST_RUNNER_EXTRA="${TEST_RUNNER_EXTRA} --timeout-factor=4" # Increase timeout because sanitizers slow down export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-gui=no --with-sanitizers=thread CC=clang-16 CXX=clang++-16 CXXFLAGS='-g' --with-boost-process" +export BITCOIN_CONFIG="--enable-zmq --with-gui=no --with-sanitizers=thread CC=clang-18 CXX=clang++-18 CXXFLAGS='-g' --with-boost-process" export CPPFLAGS="-DDEBUG_LOCKORDER -DARENA_DEBUG" export PYZMQ=true diff --git a/ci/test/00_setup_env_native_ubsan.sh b/ci/test/00_setup_env_native_ubsan.sh index 3d44490a4f102..9fb440dfd41e2 100755 --- a/ci/test/00_setup_env_native_ubsan.sh +++ b/ci/test/00_setup_env_native_ubsan.sh @@ -8,8 +8,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_ubsan -export PACKAGES="clang-16 llvm-16 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang-18 llvm-18 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" export DEP_OPTS="NO_UPNP=1 DEBUG=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports --enable-crash-hooks --with-sanitizers=undefined CC=clang-16 CXX=clang++-16" +export BITCOIN_CONFIG="--enable-zmq --enable-reduce-exports --enable-crash-hooks --with-sanitizers=undefined CC=clang-18 CXX=clang++-18" export PYZMQ=true diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 2703963a05549..b0793313fb1a5 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -11,4 +11,4 @@ export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind --timeout-factor=4" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no --enable-suppress-external-warnings CC=clang-16 CXX=clang++-16" # TODO enable GUI +export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=no --enable-suppress-external-warnings CC=clang-18 CXX=clang++-18" # TODO enable GUI diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index c71a4891b9f19..a46bde95b5584 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -19,7 +19,6 @@ fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x export RUN_UNIT_TESTS=true -export TEST_RUNNER_ENV="LC_ALL=C" export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --with-boost-process" # GUI tests disabled for now, see https://github.com/bitcoin/bitcoin/issues/23730 diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index f80a1b5174b5c..fb3f5923352e2 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" # (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) ENV APT_ARGS="-y --no-install-recommends --no-upgrade" - +SHELL ["/bin/bash", "-c"] # Install packages for i386; disabled on aarch64 and arm64 hosts RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || dpkg --add-architecture i386) RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || (apt-get update && apt-get install $APT_ARGS \ @@ -23,13 +23,11 @@ RUN apt-get update && apt-get install $APT_ARGS \ bsdmainutils \ curl \ ccache \ - clang \ cmake \ g++ \ gettext \ git \ - libc++-dev \ - libc++abi-dev \ + gnupg \ libtool \ libxcb-icccm4 \ libxcb-image0 \ @@ -42,12 +40,37 @@ RUN apt-get update && apt-get install $APT_ARGS \ libxcb-xinerama0 \ libxcb-xkb1 \ libxkbcommon-x11-0 \ - wget \ + lsb-release \ + software-properties-common \ unzip \ + wget \ m4 \ pkg-config \ zlib1g-dev +# Install Clang+LLVM and set it as default +# We don't need all packages but the default set doesn't include some +# packages we want so we will need to install some of them manually. +ARG LLVM_VERSION=18 +RUN set -ex; \ + cd /tmp && \ + wget https://apt.llvm.org/llvm.sh && \ + chmod +x llvm.sh && \ + /tmp/llvm.sh ${LLVM_VERSION} && \ + rm -rf /tmp/llvm.sh; \ + apt-get update && apt-get install $APT_ARGS \ + clang-format-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + libc++-${LLVM_VERSION}-dev \ + libc++abi-${LLVM_VERSION}-dev \ + libclang-rt-${LLVM_VERSION}-dev; \ + rm -rf /var/lib/apt/lists/*; \ + lldbUpdAltArgs="update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-${LLVM_VERSION} 100"; \ + for binName in clang clang++ clang-format clang-tidy clangd ld.lld lldb lldb-server; do \ + lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \ + done; \ + bash -c "${lldbUpdAltArgs}"; + # Python setup # PYTHON_VERSION should match the value in .python-version ARG PYTHON_VERSION=3.9.18 @@ -61,7 +84,6 @@ RUN apt-get update && apt-get install $APT_ARGS \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ - llvm \ make \ tk-dev \ xz-utils @@ -136,20 +158,6 @@ RUN \ update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; \ exit 0 -ARG LLVM_VERSION=16 -# Setup Clang+LLVM support -RUN apt-get update && apt-get install $APT_ARGS \ - lsb-release \ - software-properties-common \ - gnupg \ - && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp && \ - wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - /tmp/llvm.sh ${LLVM_VERSION} && \ - rm -rf /tmp/llvm.sh - RUN \ mkdir -p /src/dash && \ mkdir -p /cache/ccache && \ diff --git a/contrib/containers/develop/docker-compose.yml b/contrib/containers/develop/docker-compose.yml index 95241c0a563a2..9238c97183e79 100644 --- a/contrib/containers/develop/docker-compose.yml +++ b/contrib/containers/develop/docker-compose.yml @@ -1,17 +1,18 @@ -version: "3.9" services: container: entrypoint: /bin/bash build: context: '..' dockerfile: './develop/Dockerfile' - tty: true # Equivalent to -t - stdin_open: true # Equivalent to -i ports: - "9998:9998" # Mainnet Ports - "9999:9999" - "19998:19998" # Testnet Ports - "19999:19999" + security_opt: + - seccomp:unconfined + stdin_open: true # Equivalent to -i + tty: true # Equivalent to -t # A note about volumes: # diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 2333cc883107b..6cb6e53ed74e5 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -159,9 +159,15 @@ $(package)_config_opts_linux += -dbus-runtime ifneq ($(LTO),) $(package)_config_opts_linux += -ltcg endif -$(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ -ifneq (,$(findstring -stdlib=libc++,$($(1)_cxx))) -$(package)_config_opts_x86_64_linux = -xplatform linux-clang-libc++ + +ifneq (,$(findstring clang,$($(package)_cxx))) + ifneq (,$(findstring -stdlib=libc++,$($(package)_cxx))) + $(package)_config_opts_linux += -platform linux-clang-libc++ -xplatform linux-clang-libc++ + else + $(package)_config_opts_linux += -platform linux-clang -xplatform linux-clang + endif +else + $(package)_config_opts_linux += -platform linux-g++ -xplatform bitcoin-linux-g++ endif $(package)_config_opts_mingw32 = -no-opengl From 54c3b6fdb8c5101cb16cb6766a72f910ac5de4fa Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 16 Nov 2024 07:21:54 +0000 Subject: [PATCH 02/11] ci: purge package manager cache after each interaction --- contrib/containers/ci/Dockerfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index fb3f5923352e2..cbfe0541c63f1 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -46,7 +46,8 @@ RUN apt-get update && apt-get install $APT_ARGS \ wget \ m4 \ pkg-config \ - zlib1g-dev + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* # Install Clang+LLVM and set it as default # We don't need all packages but the default set doesn't include some @@ -86,7 +87,9 @@ RUN apt-get update && apt-get install $APT_ARGS \ libssl-dev \ make \ tk-dev \ - xz-utils + xz-utils \ + && rm -rf /var/lib/apt/lists/* + ENV PYENV_ROOT="/usr/local/pyenv" ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}" RUN curl https://pyenv.run | bash \ @@ -134,7 +137,8 @@ RUN apt-get update && apt-get install $APT_ARGS \ valgrind \ wine-stable \ wine64 \ - xorriso + xorriso \ + && rm -rf /var/lib/apt/lists/* ARG CPPCHECK_VERSION=2.13.0 RUN curl -sL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" | tar -xvzf - --directory /tmp/ From f851fc9d06e76b24b87e4254f109ce775c9b8e95 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 16 Nov 2024 07:44:52 +0000 Subject: [PATCH 03/11] ci: install `i386` packages only if host is `amd64`, merge layers --- contrib/containers/ci/Dockerfile | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index cbfe0541c63f1..84f4578a4187c 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -7,14 +7,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" # (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) ENV APT_ARGS="-y --no-install-recommends --no-upgrade" +# Install packages for i386 on amd64 hosts, then install common packages SHELL ["/bin/bash", "-c"] -# Install packages for i386; disabled on aarch64 and arm64 hosts -RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || dpkg --add-architecture i386) -RUN (dpkg --print-architecture | grep -Eq 'aarch64|arm64' || (apt-get update && apt-get install $APT_ARGS \ - g++-multilib \ - wine32) && rm -rf /var/lib/apt/lists/*) - -RUN apt-get update && apt-get install $APT_ARGS \ +RUN set -ex; \ + apt-get update && \ + if [[ "$(dpkg --print-architecture)" == "amd64" ]]; then \ + dpkg --add-architecture i386 && \ + apt-get update && \ + apt-get install $APT_ARGS \ + g++-multilib \ + wine32; \ + fi; \ + apt-get install $APT_ARGS \ autotools-dev \ automake \ autoconf \ From ce272ca1593b8f92fc6a1d76d8c6a57a1fb571ac Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 21 Nov 2024 06:59:21 +0000 Subject: [PATCH 04/11] ci: download and build GCC 14, allow overriding version --- contrib/containers/ci/Dockerfile | 64 ++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 84f4578a4187c..69c14508bc070 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -16,6 +16,7 @@ RUN set -ex; \ apt-get update && \ apt-get install $APT_ARGS \ g++-multilib \ + linux-libc-dev:i386 \ wine32; \ fi; \ apt-get install $APT_ARGS \ @@ -25,9 +26,11 @@ RUN set -ex; \ bison \ build-essential \ bsdmainutils \ - curl \ + ca-certificates \ ccache \ cmake \ + curl \ + flex \ g++ \ gettext \ git \ @@ -44,8 +47,10 @@ RUN set -ex; \ libxcb-xinerama0 \ libxcb-xkb1 \ libxkbcommon-x11-0 \ + linux-libc-dev \ lsb-release \ software-properties-common \ + tzdata \ unzip \ wget \ m4 \ @@ -53,6 +58,60 @@ RUN set -ex; \ zlib1g-dev \ && rm -rf /var/lib/apt/lists/* +# Compile and install GCC +ARG GCC_VERSION=14.2.0 +ARG GCC_COMPILE_ARGS="" +RUN set -ex; \ + curl -fL https://ftpmirror.gnu.org/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz -o /tmp/gcc.tar.xz; \ + mkdir -p /tmp/gcc; \ + tar -xf /tmp/gcc.tar.xz -C /tmp/gcc --strip-components=1; \ + rm /tmp/gcc.tar.xz; \ + cd /tmp/gcc; \ + ./contrib/download_prerequisites; \ + (rm *.tar.* || true); \ + dpkgArch="$(dpkg --print-architecture)"; GCC_PLATFORM_ARGS=""; \ + case "${dpkgArch##*-}" in \ + armel) GCC_PLATFORM_ARGS="--with-arch=armv4t --with-float=soft";; \ + armhf) GCC_PLATFORM_ARGS="--with-arch=armv7-a --with-float=hard --with-fpu=vfpv3-d16 --with-mode=thumb";; \ + i386|i686) GCC_PLATFORM_ARGS="--with-arch=i686";; \ + amd64) GCC_PLATFORM_ARGS="--enable-multilib --with-abi=m64 --with-arch-32=i686 --with-multilib-list=m32,m64,mx32";; \ + esac; \ + gnuArch="$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)"; \ + ln -s "/usr/include/${gnuArch}/asm" "/usr/include/asm"; \ + majorVersion="${GCC_VERSION%%.*}"; \ + ./configure \ + --build="${gnuArch}" \ + --host="${gnuArch}" \ + --target="${gnuArch}" \ + --disable-bootstrap \ + --disable-vtable-verify \ + --disable-werror \ + --enable-checking=release \ + --enable-clocale=gnu \ + --enable-default-pie \ + --enable-gnu-unique-object \ + --enable-languages=c,c++,fortran,objc,obj-c++ \ + --enable-libstdcxx-debug \ + --enable-libstdcxx-time=yes \ + --enable-libsanitizer \ + --enable-linker-build-id \ + --enable-nls \ + --enable-threads=posix \ + --prefix="/opt/gcc" \ + --program-suffix="-${majorVersion}" \ + --with-default-libstdcxx-abi=new \ + --with-gcc-major-version-only \ + --with-tune=generic \ + --without-included-gettext \ + ${GCC_COMPILE_ARGS} \ + ${GCC_PLATFORM_ARGS}; \ + make -j "$(( $(nproc) - 1 ))"; \ + make install-strip; \ + cd /tmp; \ + rm -rf /tmp/gcc; +ENV LD_LIBRARY_PATH="/opt/gcc/lib:/opt/gcc/lib64:${LD_LIBRARY_PATH}" +ENV PATH="/opt/gcc/bin:${PATH}" + # Install Clang+LLVM and set it as default # We don't need all packages but the default set doesn't include some # packages we want so we will need to install some of them manually. @@ -80,7 +139,6 @@ RUN set -ex; \ # PYTHON_VERSION should match the value in .python-version ARG PYTHON_VERSION=3.9.18 RUN apt-get update && apt-get install $APT_ARGS \ - ca-certificates \ libbz2-dev \ libffi-dev \ liblzma-dev \ @@ -158,7 +216,7 @@ ENV PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" # due to gcc-multilib installing the following symbolic link, which is needed for -m32 support. However, this causes # arm builds to also have the asm folder implicitly in the include search path. This is kind of ok, because the asm folder # for arm has precedence. -RUN ln -s x86_64-linux-gnu/asm /usr/include/asm +RUN ln -s "/usr/include/$(dpkg-architecture --query DEB_BUILD_GNU_TYPE)/asm" "/usr/include/asm" # Make sure std::thread and friends is available RUN \ From 78c140716adc0166757165d54c3f0a1ff7719bc4 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:03:35 +0000 Subject: [PATCH 05/11] ci: use distro pkgs for deps instead of building them in-tree for GCC 14 --- contrib/containers/ci/Dockerfile | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 69c14508bc070..8501747265739 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -36,6 +36,10 @@ RUN set -ex; \ git \ gnupg \ libtool \ + libgmp3-dev \ + libisl-dev \ + libmpc-dev \ + libmpfr-dev \ libxcb-icccm4 \ libxcb-image0 \ libxcb-keysyms1 \ @@ -67,8 +71,6 @@ RUN set -ex; \ tar -xf /tmp/gcc.tar.xz -C /tmp/gcc --strip-components=1; \ rm /tmp/gcc.tar.xz; \ cd /tmp/gcc; \ - ./contrib/download_prerequisites; \ - (rm *.tar.* || true); \ dpkgArch="$(dpkg --print-architecture)"; GCC_PLATFORM_ARGS=""; \ case "${dpkgArch##*-}" in \ armel) GCC_PLATFORM_ARGS="--with-arch=armv4t --with-float=soft";; \ @@ -101,6 +103,14 @@ RUN set -ex; \ --program-suffix="-${majorVersion}" \ --with-default-libstdcxx-abi=new \ --with-gcc-major-version-only \ + --with-isl-include="/usr/include" \ + --with-isl-lib="/usr/lib/${gnuArch}" \ + --with-mpc-include="/usr/include" \ + --with-mpc-lib="/usr/lib/${gnuArch}" \ + --with-mpfr-include="/usr/include" \ + --with-mpfr-lib="/usr/lib/${gnuArch}" \ + --with-gmp-include="/usr/include" \ + --with-gmp-lib="/usr/lib/${gnuArch}" \ --with-tune=generic \ --without-included-gettext \ ${GCC_COMPILE_ARGS} \ From 983ad461a8d66c27db813f2b5a166de0ece76b25 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:03:40 +0000 Subject: [PATCH 06/11] ci: drop non-C(++) langs, disable libsanitizer, NLS, drop ISL on GCC 14 Also disable debug builds of libstdcxx --- contrib/containers/ci/Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 8501747265739..d5bf5bee7e283 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -37,7 +37,6 @@ RUN set -ex; \ gnupg \ libtool \ libgmp3-dev \ - libisl-dev \ libmpc-dev \ libmpfr-dev \ libxcb-icccm4 \ @@ -86,25 +85,22 @@ RUN set -ex; \ --host="${gnuArch}" \ --target="${gnuArch}" \ --disable-bootstrap \ + --disable-libsanitizer \ + --disable-nls \ --disable-vtable-verify \ --disable-werror \ --enable-checking=release \ --enable-clocale=gnu \ --enable-default-pie \ --enable-gnu-unique-object \ - --enable-languages=c,c++,fortran,objc,obj-c++ \ - --enable-libstdcxx-debug \ + --enable-languages=c,c++ \ --enable-libstdcxx-time=yes \ - --enable-libsanitizer \ --enable-linker-build-id \ - --enable-nls \ --enable-threads=posix \ --prefix="/opt/gcc" \ --program-suffix="-${majorVersion}" \ --with-default-libstdcxx-abi=new \ --with-gcc-major-version-only \ - --with-isl-include="/usr/include" \ - --with-isl-lib="/usr/lib/${gnuArch}" \ --with-mpc-include="/usr/include" \ --with-mpc-lib="/usr/lib/${gnuArch}" \ --with-mpfr-include="/usr/include" \ @@ -113,6 +109,7 @@ RUN set -ex; \ --with-gmp-lib="/usr/lib/${gnuArch}" \ --with-tune=generic \ --without-included-gettext \ + --without-isl \ ${GCC_COMPILE_ARGS} \ ${GCC_PLATFORM_ARGS}; \ make -j "$(( $(nproc) - 1 ))"; \ From 81be90762ba0a0b50348d96abdfb86584c05327b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Sat, 16 Nov 2024 10:13:46 +0000 Subject: [PATCH 07/11] ci: split off packages needed only to build/install compilers --- contrib/containers/ci/Dockerfile | 68 +++++++++++++++++--------------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index d5bf5bee7e283..f7824377b6c75 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -4,10 +4,9 @@ FROM ubuntu:jammy ENV DEBIAN_FRONTEND="noninteractive" TZ="Europe/London" # Build and base stuff -# (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) ENV APT_ARGS="-y --no-install-recommends --no-upgrade" -# Install packages for i386 on amd64 hosts, then install common packages +# Install packages for i386 on amd64 hosts, then install base packages SHELL ["/bin/bash", "-c"] RUN set -ex; \ apt-get update && \ @@ -15,50 +14,32 @@ RUN set -ex; \ dpkg --add-architecture i386 && \ apt-get update && \ apt-get install $APT_ARGS \ - g++-multilib \ - linux-libc-dev:i386 \ - wine32; \ + gcc-multilib \ + linux-libc-dev:i386; \ fi; \ apt-get install $APT_ARGS \ autotools-dev \ automake \ autoconf \ - bison \ build-essential \ - bsdmainutils \ ca-certificates \ - ccache \ - cmake \ curl \ flex \ g++ \ - gettext \ git \ gnupg \ libtool \ libgmp3-dev \ libmpc-dev \ libmpfr-dev \ - libxcb-icccm4 \ - libxcb-image0 \ - libxcb-keysyms1 \ - libxcb-randr0 \ - libxcb-render-util0 \ - libxcb-shape0 \ - libxcb-sync1 \ - libxcb-xfixes0 \ - libxcb-xinerama0 \ - libxcb-xkb1 \ - libxkbcommon-x11-0 \ linux-libc-dev \ lsb-release \ + m4 \ + make \ + pkg-config \ software-properties-common \ tzdata \ - unzip \ wget \ - m4 \ - pkg-config \ - zlib1g-dev \ && rm -rf /var/lib/apt/lists/* # Compile and install GCC @@ -124,11 +105,7 @@ ENV PATH="/opt/gcc/bin:${PATH}" # packages we want so we will need to install some of them manually. ARG LLVM_VERSION=18 RUN set -ex; \ - cd /tmp && \ - wget https://apt.llvm.org/llvm.sh && \ - chmod +x llvm.sh && \ - /tmp/llvm.sh ${LLVM_VERSION} && \ - rm -rf /tmp/llvm.sh; \ + curl -sL https://apt.llvm.org/llvm.sh | bash -s "${LLVM_VERSION}" && \ apt-get update && apt-get install $APT_ARGS \ clang-format-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ @@ -154,7 +131,6 @@ RUN apt-get update && apt-get install $APT_ARGS \ libreadline-dev \ libsqlite3-dev \ libssl-dev \ - make \ tk-dev \ xz-utils \ && rm -rf /var/lib/apt/lists/* @@ -166,6 +142,36 @@ RUN curl https://pyenv.run | bash \ && pyenv install $PYTHON_VERSION \ && pyenv global $PYTHON_VERSION \ && pyenv rehash + +# Install packages for i386 on amd64 hosts, then install common packages +# (zlib1g-dev is needed for the Qt host binary builds, but should not be used by target binaries) +RUN set -ex; \ + apt-get update && \ + if [[ "$(dpkg --print-architecture)" == "amd64" ]]; then \ + apt-get install $APT_ARGS \ + wine32; \ + fi; \ + apt-get install $APT_ARGS \ + bison \ + bsdmainutils \ + ccache \ + cmake \ + gettext \ + libxcb-icccm4 \ + libxcb-image0 \ + libxcb-keysyms1 \ + libxcb-randr0 \ + libxcb-render-util0 \ + libxcb-shape0 \ + libxcb-sync1 \ + libxcb-xfixes0 \ + libxcb-xinerama0 \ + libxcb-xkb1 \ + libxkbcommon-x11-0 \ + unzip \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + RUN pip3 install \ codespell==1.17.1 \ flake8==3.8.3 \ From 794028c0671bba0252fc4fdc035571570b597402 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:09:36 +0000 Subject: [PATCH 08/11] ci: make sure that GDB is willing to load GCC 14's libraries --- contrib/containers/ci/Dockerfile | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index f7824377b6c75..a8a523931c0fa 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -188,14 +188,16 @@ ARG DASH_HASH_VERSION=1.4.0 RUN git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash RUN cd dash_hash && pip3 install -r requirements.txt . +# Add user with specified (or default) user/group ids +# and setup configuration files ARG USER_ID=1000 ARG GROUP_ID=1000 - -# add user with specified (or default) user/group ids -ENV USER_ID="${USER_ID}" -ENV GROUP_ID="${GROUP_ID}" -RUN groupadd -g ${GROUP_ID} dash -RUN useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash +RUN set -ex; \ + groupadd -g ${GROUP_ID} dash; \ + useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash; \ + mkdir -p /home/dash/.config/gdb; \ + echo "add-auto-load-safe-path /opt/gcc/lib:/opt/gcc/lib64" | tee /home/dash/.config/gdb/gdbinit; \ + chown ${USER_ID}:${GROUP_ID} -R /home/dash # Packages needed for all target builds RUN apt-get update && apt-get install $APT_ARGS \ From 2498839949c825981520b8efa31fdf9f25c88ddf Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Wed, 20 Nov 2024 18:10:44 +0000 Subject: [PATCH 09/11] ci: extend GDB allowlist and `LD_LIBRARY_PATH` to LLVM's libc++ --- contrib/containers/ci/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index a8a523931c0fa..781fbea59d739 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -118,6 +118,7 @@ RUN set -ex; \ lldbUpdAltArgs="${lldbUpdAltArgs} --slave /usr/bin/${binName} ${binName} /usr/bin/${binName}-${LLVM_VERSION}"; \ done; \ bash -c "${lldbUpdAltArgs}"; +ENV LD_LIBRARY_PATH="/usr/lib/llvm-${LLVM_VERSION}/lib:${LD_LIBRARY_PATH}" # Python setup # PYTHON_VERSION should match the value in .python-version @@ -196,7 +197,7 @@ RUN set -ex; \ groupadd -g ${GROUP_ID} dash; \ useradd -u ${USER_ID} -g dash -s /bin/bash -m -d /home/dash dash; \ mkdir -p /home/dash/.config/gdb; \ - echo "add-auto-load-safe-path /opt/gcc/lib:/opt/gcc/lib64" | tee /home/dash/.config/gdb/gdbinit; \ + echo "add-auto-load-safe-path /opt/gcc/lib:/opt/gcc/lib64:/usr/lib/llvm-${LLVM_VERSION}/lib" | tee /home/dash/.config/gdb/gdbinit; \ chown ${USER_ID}:${GROUP_ID} -R /home/dash # Packages needed for all target builds From 97a9b11b6b204110755481c4149f58f553f4c55e Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Fri, 8 Nov 2024 17:01:49 +0000 Subject: [PATCH 10/11] ci: don't stage packages in `/tmp`, reduce layers for `cppcheck` build --- contrib/containers/ci/Dockerfile | 34 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/contrib/containers/ci/Dockerfile b/contrib/containers/ci/Dockerfile index 781fbea59d739..efc14858eb27e 100644 --- a/contrib/containers/ci/Dockerfile +++ b/contrib/containers/ci/Dockerfile @@ -184,10 +184,28 @@ RUN pip3 install \ yq \ multiprocess -# dash_hash ARG DASH_HASH_VERSION=1.4.0 -RUN git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash -RUN cd dash_hash && pip3 install -r requirements.txt . +RUN set -ex; \ + cd /tmp; \ + git clone --depth 1 --no-tags --branch=${DASH_HASH_VERSION} https://github.com/dashpay/dash_hash; \ + cd dash_hash && pip3 install -r requirements.txt .; \ + cd .. && rm -rf dash_hash + +ARG CPPCHECK_VERSION=2.13.0 +RUN set -ex; \ + curl -fL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" -o /tmp/cppcheck.tar.gz; \ + mkdir -p /opt/cppcheck && tar -xzf /tmp/cppcheck.tar.gz -C /opt/cppcheck --strip-components=1 && rm /tmp/cppcheck.tar.gz; \ + cd /opt/cppcheck; \ + mkdir build && cd build && cmake .. && cmake --build . -j "$(( $(nproc) - 1 ))"; \ + mkdir /usr/local/share/Cppcheck && ln -s /opt/cppcheck/cfg/ /usr/local/share/Cppcheck/cfg; \ + rm -rf /tmp/cppcheck.tar.gz +ENV PATH="/opt/cppcheck/build/bin:${PATH}" + +ARG SHELLCHECK_VERSION=v0.7.1 +RUN set -ex; \ + curl -fL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" -o /tmp/shellcheck.tar.xz; \ + mkdir -p /opt/shellcheck && tar -xf /tmp/shellcheck.tar.xz -C /opt/shellcheck --strip-components=1 && rm /tmp/shellcheck.tar.xz +ENV PATH="/opt/shellcheck:${PATH}" # Add user with specified (or default) user/group ids # and setup configuration files @@ -218,16 +236,6 @@ RUN apt-get update && apt-get install $APT_ARGS \ xorriso \ && rm -rf /var/lib/apt/lists/* -ARG CPPCHECK_VERSION=2.13.0 -RUN curl -sL "https://github.com/danmar/cppcheck/archive/${CPPCHECK_VERSION}.tar.gz" | tar -xvzf - --directory /tmp/ -RUN cd /tmp/cppcheck-${CPPCHECK_VERSION} && mkdir build && cd build && cmake .. && cmake --build . -j 8 -ENV PATH="/tmp/cppcheck-${CPPCHECK_VERSION}/build/bin:${PATH}" -RUN mkdir /usr/local/share/Cppcheck && ln -s /tmp/cppcheck-${CPPCHECK_VERSION}/cfg/ /usr/local/share/Cppcheck/cfg - -ARG SHELLCHECK_VERSION=v0.7.1 -RUN curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ -ENV PATH="/tmp/shellcheck-${SHELLCHECK_VERSION}:${PATH}" - # This is a hack. It is needed because gcc-multilib and g++-multilib are conflicting with g++-arm-linux-gnueabihf. This is # due to gcc-multilib installing the following symbolic link, which is needed for -m32 support. However, this causes # arm builds to also have the asm folder implicitly in the include search path. This is kind of ok, because the asm folder From 3263879a874e96f43bcf9b9116287f2ee25c4cc6 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <63189531+kwvg@users.noreply.github.com> Date: Thu, 21 Nov 2024 05:36:40 +0000 Subject: [PATCH 11/11] ci: build nowallet depends+src using GCC 14 --- .github/workflows/matrix.json | 2 +- .gitlab-ci.yml | 2 +- ci/test/00_setup_env_native_nowallet.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/matrix.json b/.github/workflows/matrix.json index dd3cd1b4d988f..43702a4994ae9 100644 --- a/.github/workflows/matrix.json +++ b/.github/workflows/matrix.json @@ -19,7 +19,7 @@ { "depends_name": "x86_64-pc-linux-gnu_nowallet", "host": "x86_64-pc-linux-gnu", - "dep_opts": "NO_WALLET=1" + "dep_opts": "NO_WALLET=1 CC=gcc-14 CXX=g++-14" } ] }, diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 477d1f01a84dc..b335be235731d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -201,7 +201,7 @@ x86_64-pc-linux-gnu_nowallet: - .skip-in-fast-mode-template variables: HOST: x86_64-pc-linux-gnu - DEP_OPTS: "NO_WALLET=1" + DEP_OPTS: "NO_WALLET=1 CC=gcc-14 CXX=g++-14" x86_64-pc-linux-gnu_multiprocess: extends: diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 1d28e24152308..c2de81c98323d 100755 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -10,4 +10,4 @@ export CONTAINER_NAME=ci_native_nowallet export PACKAGES="python3-zmq" export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-reduce-exports --with-boost-process" +export BITCOIN_CONFIG="--enable-reduce-exports --with-boost-process CC=gcc-14 CXX=g++-14"