.gitlab-ci.yml

# This file is part of Karabo.
#
# http://www.karabo.eu
#
# Copyright (C) European XFEL GmbH Schenefeld. All rights reserved.
#
# Karabo is free software: you can redistribute it and/or modify it under
# the terms of the MPL-2 Mozilla Public License.
#
# You should have received a copy of the MPL-2 Public License along with
# Karabo. If not, see <https://www.mozilla.org/en-US/MPL/2.0/>.
#
# Karabo is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.

stages:
  - lint
  - build
  - test
  - mirror
  - systemtest

variables:
  CONAN_USER_HOME: $CI_PROJECT_DIR/.cache/conan
  PIP_CACHE_DIR:   $CI_PROJECT_DIR/.cache/pip
  PRE_COMMIT_HOME: $CI_PROJECT_DIR/.cache/pre-commit

.target_os_matrix:
  all_targets:
    matrix:
      - ARCH_TAG: ["ubuntu-20-004", "ubuntu-22-003", "ubuntu-24-002", "redhat-9-003"]
  only_redhat9:
    matrix:
      - ARCH_TAG: ["redhat-9-003"]

.build_cache:
  cache_rw:
    key: $ARCH_TAG
    paths:
      - .cache/conan/
      - .cache/pip/
      - .cache/pre-commit
    policy: pull-push
  cache_ro:
    key: $ARCH_TAG
    paths:
      - .cache/conan/
      - .cache/pip/
      - .cache/pre-commit
    policy: pull

.build_template: &setup_build_env
  cache: !reference [.build_cache, cache_rw]
  tags:
    - KaraboFramework
  image: europeanxfel/karabo-ci:$ARCH_TAG
  before_script:
    # peek inside cache
    - hostname || true
    - find /cache || true
    # try to update cmake on host (debian needs this)
    - apt-get update || true
    - apt-get install -y -qq --no-install-recommends cmake || true
    # install conan to docker image
    - |
      if type "pipx" > /dev/null; then
          # use pipx when available; needed for Ubuntu since 23.04; details at:
          # https://askubuntu.com/questions/1465218/pip-error-on-ubuntu-externally-managed-environment-%C3%97-this-environment-is-extern
          pipx install --global conan==2.5.0
      else
          python3 -m pip install --upgrade pip
          python3 -m pip install conan==2.5.0
      fi
    # do CI build
    - source set_lsb_release_info.sh
    - source ci/prepare "$CI_JOB_NAME"
  after_script:
    # clean up build directories from conan cache
    - conan remove "*" -c

.artifact_template: &artifacts-spec
  artifacts:
    name: "${CI_JOB_NAME}-${CI_COMMIT_REF_SLUG}"
    when: always
    reports:
      junit: junit*.xml
      coverage_report:
        coverage_format: cobertura
        path: coverage.xml

  coverage: '/TOTAL.+?([\d\.]+\%)/'


.services_template: &services_template
  services:
    - name: europeanxfel/existdb:2.2
      alias: existdb_host
    - name: influxdb:1.8.0
      alias: influxdb_host
  variables:
    INFLUXDB_DB: "InfluxLogTest"
    INFLUXDB_HTTP_AUTH_ENABLED: "true"
    INFLUXDB_ADMIN_USER: "infadm"
    INFLUXDB_ADMIN_PASSWORD: "admpwd"
    INFLUXDB_USER: "infusr"
    INFLUXDB_USER_PASSWORD: "usrpwd"

.image_miniconda_: &image_miniconda
  image: europeanxfel/karabo-ci:miniconda-3-006
  tags:
    - KaraboFramework

.release_build: &release-build
  stage: build
  <<: *setup_build_env
  variables:
    REL_TAG: $CI_COMMIT_REF_NAME
    FRAMEWORK_DEST_DIR: karaboFramework/tags/$CI_COMMIT_REF_NAME
    LATEST_RELEASE_TAG: latest_build
    FRAMEWORK_LATEST_DIR: karaboFramework/tags/$LATEST_RELEASE_TAG
    LATEST_TAG: $LATEST_RELEASE_TAG
    GUI_DEST_DIR: karaboGui
  script:
    - source ci/build "$CI_JOB_NAME"
  only:
    - /^(\d+\.)(\d+\.)(\d+)$/
  except:
    - branches
    - schedules

.prerelease_build: &prerelease-build
  stage: build
  <<: *setup_build_env
  variables:
    REL_TAG: $CI_COMMIT_REF_NAME
    FRAMEWORK_DEST_DIR: karaboFramework/tags/$CI_COMMIT_REF_NAME
    LATEST_PRERELEASE_TAG: latest_prerelease_build
    FRAMEWORK_LATEST_DIR: karaboFramework/tags/$LATEST_PRERELEASE_TAG
    LATEST_TAG: $LATEST_PRERELEASE_TAG
    GUI_DEST_DIR: karaboGui
  script:
    - source ci/build "$CI_JOB_NAME"
  only:
    - /^(\d+\.)(\d+\.)(\d+)((a|b|rc|\.)\d+)$/
  except:
    - branches
    - schedules

.release_build_conda: &release-build-conda
  stage: build
  variables:
    REL_TAG: $CI_COMMIT_REF_NAME
    FRAMEWORK_DEST_DIR: karaboFramework/tags/$CI_COMMIT_REF_NAME
    LATEST_RELEASE_TAG: latest_build
    FRAMEWORK_LATEST_DIR: karaboFramework/tags/$LATEST_RELEASE_TAG
    LATEST_TAG: $LATEST_RELEASE_TAG
    GUI_DEST_DIR: karaboGui
  script:
    - conda config --add channels $KARABO_CONDA_CHANNEL_URL
    - >
      python ci/miniconda/main.py karabogui
      --ci --clean
      --upload-to-mirror
      -c $KARABO_CONDA_CHANNEL
      -P $KARABO_CONDA_CHANNEL_PATH
      -C $KARABO_CONDA_CHANNEL_PATH/mirror
      --index-mirror
  only:
    - /^(\d+\.)(\d+\.)(\d+)$/
  except:
    - branches
    - schedules

.prerelease_build_conda: &prerelease-build-conda
  stage: build
  variables:
    REL_TAG: $CI_COMMIT_REF_NAME
    FRAMEWORK_DEST_DIR: karaboFramework/tags/$CI_COMMIT_REF_NAME
    LATEST_PRERELEASE_TAG: latest_prerelease_build
    FRAMEWORK_LATEST_DIR: karaboFramework/tags/$LATEST_PRERELEASE_TAG
    LATEST_TAG: $LATEST_PRERELEASE_TAG
    GUI_DEST_DIR: karaboGui
  script:
    - conda config --add channels $KARABO_CONDA_CHANNEL_URL
    - >
      python ci/miniconda/main.py karabogui
      --ci --clean
      --upload-to-mirror
      -c $KARABO_CONDA_CHANNEL
      -P $KARABO_CONDA_CHANNEL_PATH/nightly
      -C $KARABO_CONDA_CHANNEL_PATH/mirror
      --index-mirror
  only:
    - /^(\d+\.)(\d+\.)(\d+)(a|b|rc)(\d+)$/
  except:
    - branches
    - schedules

.nightly_build_conda: &nightly-build-conda
  stage: build
  variables:
    NIGHTLY_TAG: Nightly
    REL_TAG: $NIGHTLY_TAG
    FRAMEWORK_DEST_DIR: karaboFramework/nightly
    GUI_DEST_DIR: $FRAMEWORK_DEST_DIR
  script:
    - conda config --add channels $KARABO_CONDA_CHANNEL_URL
    - >
      python ci/miniconda/main.py karabogui
      --ci --clean
      --upload-to-mirror
      --nightly
      -c $KARABO_CONDA_CHANNEL
      -P $KARABO_CONDA_CHANNEL_PATH/nightly
      -C $KARABO_CONDA_CHANNEL_PATH/mirror
      --index-mirror
  only:
    - schedules

.manual_build_conda_dependency_update: &manual-build-conda-dependency-update
  stage: build
  script:
    - conda config --add channels conda-forge
    - >
      python ci/miniconda/main.py karabogui
      --ci --clean --test
      --upload-to-mirror
      -c $KARABO_CONDA_CHANNEL
      -P $KARABO_CONDA_CHANNEL_PATH/nightly
      -C $KARABO_CONDA_CHANNEL_PATH/mirror
      --index-mirror
  only:
    variables:
      - $REFRESH_MIRROR
  when: manual

.nightly_build: &nightly-build
  stage: build
  <<: *setup_build_env
  variables:
    NIGHTLY_TAG: Nightly
    REL_TAG: $NIGHTLY_TAG
    FRAMEWORK_DEST_DIR: karaboFramework/nightly
    GUI_DEST_DIR: $FRAMEWORK_DEST_DIR
  script:
    - source ci/build "$CI_JOB_NAME"

.deps_build: &deps-build
  stage: build
  <<: *setup_build_env
  script:
    - pushd extern
    - export DEPS_BASE_NAME=$LSB_RELEASE_DIST-$(echo $LSB_RELEASE_VERSION | sed -r "s/^([0-9]+).*/\1/")
    - export DEPS_DIR_ABS_PATH=$(pwd -P)/$DEPS_BASE_NAME
    - bash build.sh -q $DEPS_BASE_NAME CI
    - $DEPS_BASE_NAME/bin/python3 ./builder_path_replace.py --fix-links $DEPS_BASE_NAME/bin $DEPS_DIR_ABS_PATH __KARABO_CI_PATH__
    - $DEPS_BASE_NAME/bin/python3 ./builder_path_replace.py $DEPS_BASE_NAME/lib $DEPS_DIR_ABS_PATH __KARABO_CI_PATH__
  only:
    refs:
      - /^deps-(add|update|remove)-.+$/
      - merge_requests
    changes:
      - "extern/**/*"
  except:
    - schedules

.except_main: &except-main
  except:
    refs:
      - /^(master)$/ # the changes check will default to true
      - /^(\d+\.)(\d+\.)(X-hotfix)$/ # the changes check will default to true
      - schedules

.integration_tests: &integration-tests
  stage: test
  <<: *services_template
  <<: *artifacts-spec
  <<: *setup_build_env
  only:
    refs:
      - /^(master)$/ # the changes check will default to true
      - /^(\d+\.)(\d+\.)(X-hotfix)$/ # the changes check will default to true
      - merge_requests
    changes:
      - "src/karabo/core/**/*"
      - "src/karabo/devices/**/*"
      - "src/integrationTests/**/*"
      - "src/tools/dataLoggerIndex/**/*" # for data logger integration tests
      - "src/pythonKarabo/karabo/integration_tests/**/*"
      - "src/pythonKarabo/karabo/bound_devices/**/*"
      - "src/pythonKarabo/karabo/middlelayer_devices/**/*"
      - "src/pythonKarabo/karabo/influxdb/**/*"
      - "extern/**/*"
  except:
    refs:
      - schedules

.cpp_unit_tests: &cpp-unit-tests
  stage: test
  <<: *services_template
  <<: *artifacts-spec
  <<: *setup_build_env
  only:
    refs:
      - merge_requests
    changes:
      - "src/karabo/**/*"
      - "src/deviceServer/**/*"
      - "src/brokerMessageLogger/**/*"
      - "src/tools/brokerRates/**/*"
      - "src/tools/dataLoggerIndex/**/*"
      - "extern/**/*"
  <<: *except-main

.python_unit_tests: &python-unit-tests
  stage: test
  <<: *services_template
  <<: *artifacts-spec
  <<: *setup_build_env
  only:
    refs:
      - merge_requests
    changes:
      - "src/karabind/**/*" # bound api new binding code
      - "src/karabo/**/*" # bound api depends on the c++ build
      - "src/pythonKarabo/**/*"
      - "extern/**/*"
  <<: *except-main

# GUI tests to be executed in a Conda environment
.gui_unit_tests: &gui-unit-tests
  stage: test
  script:
    - conda config --add channels $KARABO_CONDA_CHANNEL_URL
    - >
      python ci/miniconda/main.py karabogui
      -c $KARABO_CONDA_CHANNEL
      --ci --clean --test
      --skip-build
  only:
    refs:
      - merge_requests
    changes:
      - "src/pythonGui/**/*"
      - "src/pythonKarabo/karabo/native/**/*"
      - "src/pythonKarabo/karabo/common/**/*"
      - "conda-recipes/karabo-native/**/*"
      - "conda-recipes/karabogui/**/*"
  <<: *artifacts-spec

### Jobs

# static source code analysis
pre-commit-check:
  stage: lint
  image: python:3.12
  before_script:
    - find /cache || true
  script:
    - pip install pre-commit
    - pre-commit run --all-files
  cache: !reference [.build_cache, cache_rw]
  only:
    refs:
      - merge_requests
    changes:
      - "src/**/*"
  allow_failure: false


test_docs:
  stage: lint
  image: europeanxfel/karabo-ci:miniconda-3lint-003
  script:
    - pushd doc
    - pip install -r requirements.txt
    - conda config --add channels conda-forge
    - conda install doxygen
    - make doctest
  only:
    refs:
      - merge_requests
    changes:
      - "src/**/*"
      - "doc/**/*"

systemtest:
  stage: systemtest
  image: europeanxfel/karabo-ci:miniconda-3-002
  allow_failure: true
  only:
    - /^(\d+\.)(\d+\.)(\d+)((a|b|rc)(\d+))?$/
  script:
    - python ci/portal/portal_start.py

# release build

release build:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *release-build

release build miniconda:
  <<: *image_miniconda
  <<: *release-build-conda

release build miniconda-osx64:
  before_script:
    - ci/miniconda/cleanup.sh
  tags:
    - macos
  <<: *release-build-conda

release build miniconda-win64:
  before_script:
    - ci/miniconda/cleanup.cmd
  tags:
    - Win10
  <<: *release-build-conda

# prerelease build

prerelease build:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *prerelease-build

prerelease build miniconda:
  <<: *image_miniconda
  <<: *prerelease-build-conda

prerelease build miniconda-osx64:
  before_script:
    - ci/miniconda/cleanup.sh
  tags:
    - macos
  <<: *prerelease-build-conda

prerelease build miniconda-win64:
  before_script:
    - ci/miniconda/cleanup.cmd
  tags:
    - Win10
  <<: *prerelease-build-conda

# nightly release build

nightly build:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *nightly-build
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_JOB == "nightly-build"

nightly build miniconda-osx64:
  before_script:
    - ci/miniconda/cleanup.sh
  tags:
    - macos
  <<: *nightly-build-conda

nightly build miniconda-win64:
  before_script:
    - ci/miniconda/cleanup.cmd
  tags:
    - Win10
  <<: *nightly-build-conda

nightly build miniconda:
  <<: *image_miniconda
  <<: *nightly-build-conda

# Dependencies build

deps build:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *deps-build

# MR test builds

python_unit tests:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *python-unit-tests
  script:
    - bash ./auto_build_all.sh Debug --runTests --skipCppTests

cpp_unit tests:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *cpp-unit-tests
  script:
    - bash ./auto_build_all.sh Debug --runTests --skipPythonTests

gui_unit tests miniconda:
  <<: *image_miniconda
  <<: *gui-unit-tests

gui_unit tests miniconda-osx64:
  before_script:
    - ci/miniconda/cleanup.sh
  tags:
    - macos
  <<: *gui-unit-tests

gui_unit tests miniconda-win64:
  before_script:
    - ci/miniconda/cleanup.cmd
  tags:
    - Win10
  <<: *gui-unit-tests

gui_dependency_updates build miniconda:
  <<: *image_miniconda
  <<: *manual-build-conda-dependency-update

gui_dependency_updates build miniconda-osx64:
  before_script:
    - ci/miniconda/cleanup.sh
  tags:
    - macos
  <<: *manual-build-conda-dependency-update

gui_dependency_updates build miniconda-win64:
  before_script:
    - ci/miniconda/cleanup.cmd
  tags:
    - Win10
  <<: *manual-build-conda-dependency-update

# Post-merge integration test builds

python_integration tests:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *integration-tests
  script:
    - bash ./auto_build_all.sh Debug --runIntegrationTests --skipCppTests

cpp_integration tests:
  parallel: !reference [.target_os_matrix, all_targets]
  <<: *integration-tests
  script:
    - bash ./auto_build_all.sh Debug --runIntegrationTests --skipPythonTests

# Scheduled coverage tests

coverage tests:
  parallel: !reference [.target_os_matrix, only_redhat9]
  stage: test
  <<: *services_template
  <<: *artifacts-spec
  <<: *setup_build_env
  script:
    - bash ./auto_build_all.sh CodeCoverage --quiet
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_JOB == "coverage-tests"

# Scheduled long tests

long tests:
  parallel: !reference [.target_os_matrix, all_targets]
  stage: test
  <<: *services_template
  <<: *artifacts-spec
  <<: *setup_build_env
  script:
    - bash ./auto_build_all.sh Debug --runLongTests --quiet
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - "src/cppLongTests/**/*"
    - if: $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_JOB == "long-tests"

# Publish to the GitHub mirror - 1st. of 2 steps
#
# 1st. step is triggered by release tags automatically and publishes the
# master branch and the tags of the Framework to an internal mirror. As part
# of this step, the git commit history is edited using git-filter-repo to
# remove personal data from contributors who do not want their personal data
# exposed in the public GitHub mirror.
#
# The 2nd. step will be manual, triggered from the internal mirror on demand
# and will be responsible for updating the public GitHub mirror with the
# contents of the internal mirror.
internal_mirror_publish:
  stage: mirror
  image: europeanxfel/karabo-ci:miniconda-3-006

  only:
    - /^(\d+\.)(\d+\.)(\d+)((a|b|rc)(\d+))?$/
  except:
    - branches
    - schedules
  script:

    # Retrieves the mail and name mappings for authors that have chosen to keep their data hidden in the
    # commit history of the public repository.
    - git clone https://gitlab-ci-token:$CI_JOB_TOKEN@git.xfel.eu/Karabo/contriboptouts.git
    # Retrieves the master branch and the tags of the Karabo Framework repository.
    - git clone https://gitlab-ci-token:$CI_JOB_TOKEN@git.xfel.eu/Karabo/Framework.git --branch master --single-branch FrameworkHead
    - cd FrameworkHead
    # To be on the safe side and prevent any push to the origin, remove it from the list of remotes (this is what git-filter-repo does).
    - git remote rm origin
    # Renames master branch to main and pushes main and tags to the internal mirror.
    # The KARABO_FRAMEWORK_MIRROR_TOKEN must be an environment variable defined in the Framework Gitlab project whose
    # value is a project access token generated for the FrameworkMirror Gitlab project.
    - git remote add internal_mirror https://oauth2:$KARABO_FRAMEWORK_MIRROR_TOKEN@git.xfel.eu/Karabo/FrameworkMirror.git
    # Move the master branch to main (keep history)
    - git branch -m main
    - git push --mirror internal_mirror
    # Removes the local copy of the Karabo Framework repository - all the filtering will happen on the internal mirror.
    - cd ..
    - rm -rf FrameworkHead
    # The removal of files above 100 Mb (GitHub imposed limit) and the obfuscation of opts out must be performed over the just
    # updated internal mirror to avoid any conflicts during the push.
    - git clone https://oauth2:$KARABO_FRAMEWORK_MIRROR_TOKEN@git.xfel.eu/Karabo/FrameworkMirror.git
    - cd FrameworkMirror
    - git-filter-repo --invert-paths --path src/environment.in/KARABO_BROKER
    - git-filter-repo --strip-blobs-bigger-than 100M
    # in the conversion from SVN to git, email addresses were not correctly preserved,
    # rather we have, username@some-uuid-token. Here we map known users to correct email
    # addresses. Since the commits affected by this are all long in the past, a static definition
    # of the mapping suffices.
    - |
      git-filter-repo --email-callback 'import yaml;
      f=open("../contriboptouts/karabo-svn-fixer.yaml");
      mapping=yaml.safe_load(f);
      f.close();
      user, _ = email.decode().split("@");
      return mapping.get(user, email.decode()).encode()'
    # we filter mails now
    - git-filter-repo --mailmap ../contriboptouts/karabo-mailmap
    # As git-filter-repo for safety reasons removes the "origin" remote, we set again the new remote,
    # "internal_mirror" (a remote with the same name had been defined a few steps back, but for another local).
    # Details about how git-filter-repo works (the removal of the origin remote is the third step) can be
    # found at https://htmlpreview.github.io/?https://github.com/newren/git-filter-repo/blob/docs/html/git-filter-repo.html#INTERNALS.
    - git remote add internal_mirror https://oauth2:$KARABO_FRAMEWORK_MIRROR_TOKEN@git.xfel.eu/Karabo/FrameworkMirror.git
    # Note: the push below requires the FrameworkMirror repository to be set to allow force push to its main branch (by default not allowed)
    - git push -u internal_mirror main --force --prune
    # Note: the forced push of the tags below is needed because 'git-filter-repo' changes the commits SHAs of the repo
    # and updates the tags to refer to the new commit identifiers. If the tags are not pushed they will still refer
    # to the old commit idenfitifiers in the mirror.
    - git push internal_mirror --tags --force