.gitlab-ci.yml

# MIT License
#
# Copyright (c) 2017-2023 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

include:
  - project: 'amd/ci-templates'
    ref: main
    file:
      - /defaults.yaml
      - /deps-cmake.yaml
      - /deps-rocm.yaml
      - /deps-vcpkg.yaml
      - /gpus-rocm.yaml
      - /rules.yaml

stages:
  - lint
  - autotune
  - build
  - test
  - benchmark

variables:
  PACKAGE_DIR: $BUILD_DIR/package
  AUTOTUNE_CONFIG_DIR: ${CI_PROJECT_DIR}/autotune_config

clang-format:
  extends:
    - .deps:rocm
  stage: lint
  needs: []
  tags:
    - rocm-build
  variables:
    CLANG_FORMAT: "/opt/rocm/llvm/bin/clang-format"
    GIT_CLANG_FORMAT: "/opt/rocm/llvm/bin/git-clang-format"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
  script:
    - cd $CI_PROJECT_DIR
    - git config --global --add safe.directory $CI_PROJECT_DIR
    - scripts/code-format/check-format.sh $CI_MERGE_REQUEST_DIFF_BASE_SHA --binary "$CLANG_FORMAT"

.cmake-minimum-vcpkg:
  extends:
    - .deps:rocm
    - .deps:cmake-minimum
    - .deps:vcpkg
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-minimum", before_script]
    - !reference [".deps:vcpkg", before_script]
    - $VCPKG_DIR/vcpkg install
        gtest
        benchmark

.cmake-minimum-apt:
  extends:
    - .deps:rocm
    - .deps:cmake-minimum
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-minimum", before_script]
    - $SUDO_CMD apt-get install -y -qq
        libgtest-dev
        libbenchmark-dev

.build:vcpkg-apt:
  stage: build
  tags:
    - rocm-build
  extends:
    - .gpus:rocm-gpus
    - .rules:build
# Missing -Werror and other diagnostic flags due to rocm-terminal sporting an old googletest APT package (Ubuntu 18.04).
# Here we're only testing the consumption logic, and we want to avoid new errors breaking logic testing
  script:
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      "$(if [ -n "$VCPKG_DIR" ]; then echo "-DCMAKE_TOOLCHAIN_FILE=$VCPKG_DIR/scripts/buildsystems/vcpkg.cmake"; fi)"
      -D BUILD_TEST=ON
      -D BUILD_EXAMPLE=ON
      -D BUILD_BENCHMARK=ON
      -D GPU_TARGETS=$GPU_TARGETS
      -D AMDGPU_TEST_TARGETS=$GPU_TARGETS
      -S $CI_PROJECT_DIR
      -B $BUILD_DIR
    - cmake
      --build $BUILD_DIR
      --target test_basic

build:cmake-minimum-vcpkg:
  stage: build
  needs: []
  extends:
    - .cmake-minimum-vcpkg
    - .build:vcpkg-apt

build:cmake-minimum-apt:
  stage: build
  needs: []
  extends:
    - .cmake-minimum-apt
    - .build:vcpkg-apt

.cmake-latest:
  extends:
    - .deps:rocm
    - .deps:cmake-latest
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-latest", before_script]

.cmake-minimum:
  extends:
    - .deps:rocm
    - .deps:cmake-minimum
  before_script:
    - !reference [".deps:rocm", before_script]
    - !reference [".deps:cmake-minimum", before_script]

.build:common:
  stage: build
  tags:
    - rocm-build
  extends:
    - .gpus:rocm-gpus
    - .rules:build
  script:
    - mkdir -p $BUILD_DIR
    - cd $BUILD_DIR
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_CXX_FLAGS="-Wall -Wextra -Werror"
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=ON
      -D BUILD_EXAMPLE=ON
      -D BUILD_BENCHMARK=OFF
      -D GPU_TARGETS=$GPU_TARGETS
      -D AMDGPU_TEST_TARGETS=$GPU_TARGETS
      -S $CI_PROJECT_DIR
      -B $BUILD_DIR
    - cmake --build $BUILD_DIR
  artifacts:
    paths:
      - $BUILD_DIR/test/test_*
      - $BUILD_DIR/test/rocprim/test_*
      - $BUILD_DIR/test/CTestTestfile.cmake
      - $BUILD_DIR/test/rocprim/CTestTestfile.cmake
      - $BUILD_DIR/gtest/
      - $BUILD_DIR/CMakeCache.txt
      - $BUILD_DIR/.ninja_log
      - $BUILD_DIR/CTestTestfile.cmake
    expire_in: 2 weeks

build:cmake-latest:
  stage: build
  needs: []
  extends:
    - .cmake-latest
    - .build:common

build:cmake-minimum:
  stage: build
  needs: []
  extends:
    - .cmake-minimum
    - .build:common

build:package:
  stage: build
  needs: []
  tags:
    - rocm-build
  extends:
    - .cmake-minimum
    - .gpus:rocm-gpus
    - .rules:build
  script:
    - mkdir -p $PACKAGE_DIR
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      -B $PACKAGE_DIR
      -S $CI_PROJECT_DIR
    - cd $PACKAGE_DIR
    - cpack -G "DEB;ZIP"
  artifacts:
    paths:
      - $PACKAGE_DIR/rocprim*.deb
      - $PACKAGE_DIR/rocprim*.zip
    expire_in: 2 weeks

build:benchmark:
  stage: build
  needs:
    - job: "autotune:generate-config"
      optional: true
  tags:
    - rocm-build
  extends:
    - .cmake-minimum
    - .gpus:rocm-gpus
    - .rules:build
  script:
# If we have a custom config created by autotune:create-config
    - "[ -d ${AUTOTUNE_CONFIG_DIR} ] && cp -r -f ${AUTOTUNE_CONFIG_DIR}/* ${CI_PROJECT_DIR}/"
    - mkdir -p $BUILD_DIR
    - cd $BUILD_DIR
    - cmake
      -B $BUILD_DIR
      -S $CI_PROJECT_DIR
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=OFF
      -D BUILD_EXAMPLE=OFF
      -D BUILD_BENCHMARK=ON
      -D GPU_TARGETS=$GPU_TARGETS
    - cmake --build .
  artifacts:
    paths:
      - $BUILD_DIR/benchmark/*
      - $BUILD_DIR/.ninja_log
      - $BUILD_DIR/deps/googlebenchmark/
    expire_in: 2 weeks

autotune:build:
  stage: autotune
  needs: []
  tags:
    - rocm-build
  extends:
    - .cmake-minimum
    - .gpus:rocm-gpus
    - .rules:manual
  variables:
    BENCHMARK_TARGETS: benchmark_config_tuning
  script:
    - mkdir -p $BUILD_DIR
    - cd $BUILD_DIR
    - 'printf "Building benchmark targets: %s\n" "$BENCHMARK_TARGETS"'
    - cmake
      -B $BUILD_DIR
      -S $CI_PROJECT_DIR
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      -D BUILD_TEST=OFF
      -D BUILD_EXAMPLE=OFF
      -D BUILD_BENCHMARK=ON
      -D BENCHMARK_CONFIG_TUNING=ON
      -D GPU_TARGETS=$GPU_TARGETS
    - cmake --build . --target $BENCHMARK_TARGETS
    - 'rm -rf $BUILD_DIR/benchmark/benchmark*.parallel'
  artifacts:
    paths:
      - $BUILD_DIR/benchmark/benchmark*
      - $BUILD_DIR/.ninja_log
      - $BUILD_DIR/deps/googlebenchmark/
    expire_in: 1 week

test:
  stage: test
  extends:
    - .cmake-minimum
    - .rules:test
    - .gpus:rocm
  needs:
    - build:cmake-minimum
  script:
    - cd $BUILD_DIR
    - cmake
      -D CMAKE_PREFIX_PATH=/opt/rocm
      -P $CI_PROJECT_DIR/cmake/GenerateResourceSpec.cmake
    - cat ./resources.json
    - ctest
      --output-on-failure
      --repeat-until-fail 2
      --tests-regex "hip|$GPU_TARGET"
      --resource-spec-file ./resources.json
      --parallel $PARALLEL_JOBS

.test-package:
  script:
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      -D GPU_TARGETS=$GPU_TARGETS
      -S "$CI_PROJECT_DIR/test/extra"
      -B "$CI_PROJECT_DIR/package_test"
    - cmake --build "$CI_PROJECT_DIR/package_test"
    - "$CI_PROJECT_DIR/package_test/test_rocprim_package"
    - cd "$CI_PROJECT_DIR/package_test"
    - ctest --output-on-failure --repeat-until-fail 2

test:install:
  stage: test
  needs: []
  tags:
    - rocm
  extends:
    - .cmake-minimum
    - .rules:test
    - .gpus:rocm-gpus
  script:
    - cmake
      -G Ninja
      -D CMAKE_CXX_COMPILER="$AMDCLANG"
      -D CMAKE_BUILD_TYPE=Release
      -B build
      -S $CI_PROJECT_DIR
    - $SUDO_CMD cmake --build build --target install
    - !reference [.test-package, script]

test:deb:
  stage: test
  needs:
    - build:package
  tags:
    - rocm
  extends:
    - .cmake-minimum
    - .rules:test
    - .gpus:rocm-gpus
  script:
    - $SUDO_CMD dpkg -i $PACKAGE_DIR/rocprim*.deb
    - !reference [.test-package, script]

test:docs:
  stage: test
  extends: .rules:test
  image: sphinxdoc/sphinx-latexpdf:latest
  needs: []
  before_script:
    - apt-get update -qq
    - apt-get install -y -qq doxygen
    - pip3 install --user -r $CI_PROJECT_DIR/docs/requirements.txt
  script:
    - cd docs
    - make
  artifacts:
    paths:
      - $CI_PROJECT_DIR/docs/_build
      - $CI_PROJECT_DIR/docs/docBin
    expire_in: 1 week

.benchmark-base:
  stage: benchmark
  extends:
    - .rules:benchmark
  variables:
    BENCHMARK_RESULT_DIR: ${CI_PROJECT_DIR}/benchmark_results
    BENCHMARK_RESULT_CACHE_DIR: ${BENCHMARK_RESULT_DIR}_cache

benchmark:
  needs:
    - build:benchmark
  extends:
    - .cmake-minimum
    - .gpus:rocm
    - .benchmark-base
  variables:
    BENCHMARK_FILENAME_REGEX: ^benchmark
    BENCHMARK_ALGORITHM_REGEX: "benchmark_device_merge_sort_block_sort"
  script:
    - 'printf "CI Variables used in benchmarks:\nBENCHMARK_RESULT_DIR: %s\nBENCHMARK_FILENAME_REGEX: %s\nBENCHMARK_ALGORITHM_REGEX: %s \n" "$BENCHMARK_RESULT_DIR" "$BENCHMARK_FILENAME_REGEX" "$BENCHMARK_ALGORITHM_REGEX"'
    - cd "${CI_PROJECT_DIR}"
    - mkdir -p "${BENCHMARK_RESULT_DIR}"
    - python3
      .gitlab/run_benchmarks.py
      --benchmark_dir "${BUILD_DIR}/benchmark"
      --benchmark_gpu_architecture "${GPU_TARGET}"
      --benchmark_output_dir "${BENCHMARK_RESULT_DIR}"
      --benchmark_filename_regex "${BENCHMARK_FILENAME_REGEX}"
      --benchmark_filter_regex "${BENCHMARK_ALGORITHM_REGEX}"
  artifacts:
    paths:
      - ${BENCHMARK_RESULT_DIR}
    expire_in: 1 week

benchmark:cache-or-report:
  needs:
    - benchmark
  extends:
    - .benchmark-base
  tags:
    - single-cache
  cache:
    key: benchmark-cache
    paths:
      - ${BENCHMARK_RESULT_CACHE_DIR}
  script:
    # If on MR branch, generate report, else cache results
    - >
      if [ ! -z "${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}" ]; then
        if [ ! -d "${BENCHMARK_RESULT_CACHE_DIR}" ]; then
          echo 'ERROR: Cache directory does not exist'
          exit 1
        elif [ ! -d "${BENCHMARK_RESULT_DIR}" ]; then
          echo 'ERROR: Benchmark results directory does not exist'
          exit 1
        else
          echo 'INFO: Files in cache (reference benchmarks):'
          ls -al ${BENCHMARK_RESULT_CACHE_DIR}
          echo 'INFO: Generating report...'
          python3 .gitlab/generate_report.py --old ${BENCHMARK_RESULT_CACHE_DIR} --new ${BENCHMARK_RESULT_DIR}
        fi
      elif [ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]; then
        echo 'INFO: Caching benchmark results...'
        mkdir -p ${BENCHMARK_RESULT_CACHE_DIR}
        cp -R ${BENCHMARK_RESULT_DIR}/*.json ${BENCHMARK_RESULT_CACHE_DIR}
      else
        echo 'ERROR: Neither on a merge-request branch or the default branch'
        exit 1
      fi


.autotune-base:
  stage: autotune
  extends:
    - .rules:manual
  variables:
    AUTOTUNE_RESULT_DIR: ${CI_PROJECT_DIR}/autotune_results

autotune:execute-tuning:
  needs:
    - autotune:build
  extends:
    - .autotune-base
    - .cmake-minimum
    - .gpus:rocm
  variables:
    AUTOTUNE_FILENAME_REGEX: ^benchmark
    AUTOTUNE_ALGORITHM_REGEX: ""
  timeout: 8h
  artifacts:
    paths:
      - ${AUTOTUNE_RESULT_DIR}/*.json
  script:
    - 'printf "CI Variables used in benchmarks:\nAUTOTUNE_RESULT_DIR: %s\nAUTOTUNE_FILENAME_REGEX: %s\nAUTOTUNE_ALGORITHM_REGEX: %s \n" "$AUTOTUNE_RESULT_DIR" "$AUTOTUNE_FILENAME_REGEX" "$AUTOTUNE_ALGORITHM_REGEX"'
    - cd "${CI_PROJECT_DIR}"
    - mkdir -p "${AUTOTUNE_RESULT_DIR}"
    - python3
      .gitlab/run_benchmarks.py
      --benchmark_dir "${BUILD_DIR}/benchmark"
      --benchmark_gpu_architecture "${GPU_TARGET}"
      --benchmark_output_dir "${AUTOTUNE_RESULT_DIR}"
      --benchmark_filename_regex "${AUTOTUNE_FILENAME_REGEX}"
      --benchmark_filter_regex "${AUTOTUNE_ALGORITHM_REGEX}"

autotune:generate-config:
  image: python:3.10.5-buster
  needs:
    - job: "autotune:execute-tuning"
      optional: true
  extends:
    - .rules:manual
    - .autotune-base
  variables:
    AUTOTUNE_CONFIG_REPO_PATH: /rocprim/include/rocprim/device/detail/config
    AUTOTUNE_RESULT_CACHE_DIR: ${AUTOTUNE_RESULT_DIR}_cache
  tags:
    - single-cache
  cache:
    key: autotune-cache
    paths:
      - autotune_results_cache/
  script:
    # Set cache dir variables depending on if this is a MR or not
    - >
      if [ ! -z "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]; then
        AUTOTUNE_RESULT_CACHE_BRANCH_DIR="${AUTOTUNE_RESULT_CACHE_DIR}/${CI_MERGE_REQUEST_SOURCE_BRANCH_NAME}"
        AUTOTUNE_RESULT_CACHE_TARGET_BRANCH_DIR="${AUTOTUNE_RESULT_CACHE_DIR}/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
      else
        AUTOTUNE_RESULT_CACHE_BRANCH_DIR="${AUTOTUNE_RESULT_CACHE_DIR}/${CI_COMMIT_BRANCH}"
      fi
    # If the global cache dir does not exist, create it
    - mkdir -p $AUTOTUNE_RESULT_CACHE_DIR
    # If there are fresh results in the artifacts, cache them in the branch cache
    # If there are no fresh results, check branch cache
    # If there are no branch cache results, check TARGET branch cache
    # If there are TARGET branch cache results, cache them in the branch cache
    - >
      if [ -d "$AUTOTUNE_RESULT_DIR" ]; then
        mkdir -p $AUTOTUNE_RESULT_CACHE_BRANCH_DIR
        cp -R -u ${AUTOTUNE_RESULT_DIR}/*.json ${AUTOTUNE_RESULT_CACHE_BRANCH_DIR}
      elif [ -d "$AUTOTUNE_RESULT_CACHE_BRANCH_DIR" ]; then
        mkdir -p $AUTOTUNE_RESULT_DIR
        cp -R -u ${AUTOTUNE_RESULT_CACHE_BRANCH_DIR}/*.json ${AUTOTUNE_RESULT_DIR}
      elif [ -d "$AUTOTUNE_RESULT_CACHE_TARGET_BRANCH_DIR" ]; then
        mkdir -p $AUTOTUNE_RESULT_DIR
        cp -R -u ${AUTOTUNE_RESULT_CACHE_TARGET_BRANCH_DIR}/*.json ${AUTOTUNE_RESULT_DIR}
        mkdir -p $AUTOTUNE_RESULT_CACHE_BRANCH_DIR
        cp -R -u ${AUTOTUNE_RESULT_DIR}/*.json ${AUTOTUNE_RESULT_CACHE_BRANCH_DIR}
      else
        echo 'ERROR: No autotune results found in previous artifacts, the branch cache or the target branch cache...'
        exit 1
      fi
    # List the final .json files to use for config generation
    - ls -al ${AUTOTUNE_RESULT_DIR}
    - cd "${CI_PROJECT_DIR}"
    - python3 -m pip install jinja2
    - mkdir -p ${AUTOTUNE_CONFIG_DIR}${AUTOTUNE_CONFIG_REPO_PATH}
    - python3
      scripts/autotune/create_optimization.py
      --benchmark_files ${AUTOTUNE_RESULT_DIR}/*.json
      --out_basedir "${AUTOTUNE_CONFIG_DIR}${AUTOTUNE_CONFIG_REPO_PATH}"
  artifacts:
    paths:
      - ${AUTOTUNE_CONFIG_DIR}

scheduled-check-changes:
  stage: autotune
  extends: .rules:scheduled-check-changes