.gitlab-ci.yml

---
default:
  # we use this tag to select the runner to be used for a given job
  tags:
    - crab3

variables:
  IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"  # to distinct it from commit tag and final image tag
  RELEASE_IMAGE_TAG: "${CI_COMMIT_TAG}-stable" # final tag name, e.g., v3.240904-stable
  # The `DOCKER_TLS_CERTDIR` variables is needed to run Docker-in-Docker, `DOCKER_BUILDKIT` is to make sure the docker build use the new BuildKit.
  # https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#docker-in-docker-with-tls-enabled-in-the-docker-executor
  # Creating a docker context is required to be able to cache to the registry using Buildkit.
  # https://docs.docker.com/build/cache/backends/
  DOCKER_TLS_CERTDIR: "/certs"
  DOCKER_BUILDKIT: 1
  CHECK_TEST_RESULT: "false"

# This key define variables which are later used in `!reference` tag in `rules`.
# Ref https://docs.gitlab.com/ee/ci/jobs/index.html#hide-jobs
# and https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
.default_rules:
  default:
    - if: $CI_COMMIT_TAG =~ /pypi-.*/         # match tag: pypi-(prod|preprod|test*)-1714418922
  release:
    - if: $CI_COMMIT_TAG =~ /v3\.[0-9]{6}.*/  # match tag: v3.240101
  skip_all_base:
    - if: $BUILD || $BUILD_DEPLOY || $ONLY_BUILD_RELEASE || $SUBMIT_STATUS_TRACKING || $MANUAL_CI_PIPELINE_ID
      when: never                             # helper var

stages:
  - prepare_env
  - prepare_release
  - build_docker
  - deploy
  - submit_testsuite
  - run_testsuite_CV
  - run_testsuite_CCV
  - check_testsuite
  - tagging_release

get_env:
  # NB rules are evaluated like python `any()`. If there is no rule the job runs all of the times.
  rules:
    # !reference in following lines tells GITLAB to replace with the value of the indicated variables.
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: prepare_env
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - printenv  # debug check ci env
    - cicd/gitlab/parseEnv.sh $CI_COMMIT_TAG  # create .env
  artifacts:
    paths:
      - .env
    expire_in: 1 week

set_version_name:
  rules:
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: prepare_release
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - |
        echo -e "\n__version__ = \"${CI_COMMIT_TAG}\" #Automatically added during build process\n" >> src/python/TaskWorker/__init__.py;
    - |
        echo -e "\n__version__ = \"${CI_COMMIT_TAG}\" #Automatically added during build process\n" >> src/python/CRABInterface/__init__.py;
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
        - src/python/CRABInterface/__init__.py
      policy: push

build_rest_image:
  rules:
    - if: $BUILD || $BUILD_DEPLOY || $ONLY_BUILD_RELEASE
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  before_script:
    - docker info
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push -f "${CI_PROJECT_DIR}/cicd/crabserver_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabserver:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabserver:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG}" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/CRABInterface/__init__.py
      policy: pull

build_tw_image:
  rules:
    - if: $BUILD || $BUILD_DEPLOY || $ONLY_BUILD_RELEASE
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push -f "${CI_PROJECT_DIR}/cicd/crabtaskworker_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtaskworker:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtaskworker:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG}" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_monit_image:
  rules:
    - if: $BUILD
    - !reference [.default_rules, release]
  stage: build_docker
  needs: ["build_tw_image", "get_env"]
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/monit_pypi/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtwmonit:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtwmonit:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtwmonit:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabtwmonit:v3.latest" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_crabtwfilebeat_image:
  rules:
    - if: $BUILD
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/filebeat/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabtwfilebeat:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabtwfilebeat:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabtwfilebeat:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabtwmonit:v3.latest" .
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - src/python/TaskWorker/__init__.py
      policy: pull

build_spark_image:
  rules:
    - if: $BUILD
    - !reference [.default_rules, release]
  stage: build_docker
  image:
    name: docker:27.1.1
  services:
    - name: docker:27.1.1-dind
  script:
    - docker login -u $CMSCRAB_REGISTRY_USER -p $CMSCRAB_REGISTRY_PASSWORD $CMSCRAB_REGISTRY_URL
    - source .env
    - docker context create mycontext
    - docker buildx create mycontext --use --name mybuilder --bootstrap
    - docker buildx build --push --build-arg="BASE_TAG=${IMAGE_TAG}" -f "${CI_PROJECT_DIR}/cicd/monit_spark/Dockerfile" --cache-to=type=registry,ref="registry.cern.ch/cmscrab/crabspark:pypi-${REST_Instance}-cache",image-manifest=true,mode=max --cache-from=type=registry,ref="registry.cern.ch/cmscrab/crabspark:pypi-${REST_Instance}-cache" -t "registry.cern.ch/cmscrab/crabspark:${IMAGE_TAG}" -t "registry.cern.ch/cmscrab/crabspark:v3.latest" .


deploy_server:
  rules:
    - if: $BUILD_DEPLOY
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: deploy
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    # load KUBECONTEXT
    - source .env
    - echo $KUBECONFIG_FILE
    - yq -i -y ".\"current-context\" = \"$KUBECONTEXT\"" $KUBECONFIG_FILE
    - export KUBECONFIG=$KUBECONFIG_FILE
    - kubectl set image deployment/crabserver "crabserver=registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG}"
    - kubectl rollout status deployment crabserver --watch --timeout=5m

.deploy_tw_template:
  rules:
    - if: $BUILD_DEPLOY
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: deploy
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  script:
    - source .env
    - export Environment # from .env
    - export Service="${SERVICE}"
    - export Image="${IMAGE_TAG}"
    - export SSH_KEY=$(cicd/gitlab/credFile.sh $CRAB_TW_SSH_KEY ssh)
    - bash -x cicd/gitlab/deployTW.sh

deploy_taskworker:
  extends: .deploy_tw_template
  variables:
    SERVICE: TaskWorker

deploy_publisher_schedd:
  extends: .deploy_tw_template
  variables:
    SERVICE: Publisher_schedd

deploy_publisher_rucio:
  extends: .deploy_tw_template
  variables:
    SERVICE: Publisher_rucio

task_submission:
  rules:
    - if: $SUBMIT_STATUS_TRACKING || $CLIENT_VALIDATION_SUITE || $CLIENT_CONFIGURATION_VALIDATION
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: submit_testsuite
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=CMSSW_13_0_2
    - export Task_Submission_Status_Tracking=true
    - export Client_Configuration_Validation=true
    - export Client_Validation_Suite=true
    - bash -x cicd/gitlab/executeTests.sh
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - workdir/submitted_tasks_CV
        - workdir/submitted_tasks_CCV
        - workdir/submitted_tasks_TS
      policy: push
    - key: submitted_tasks_TS_latest
      paths:
        - workdir/submitted_tasks_TS
      policy: push
    - key: submitted_tasks_CV_latest
      paths:
        - workdir/submitted_tasks_CV
      policy: push
    - key: submitted_tasks_CCV_latest
      paths:
        - workdir/submitted_tasks_CCV
      policy: push

client_validation_suite:
  rules:
    - if: $MANUAL_CI_PIPELINE_ID
      when: never
    - if: $CLIENT_VALIDATION_SUITE
    - if: $SKIP_DEPLOY || $FORCE_RELEASE
      when: never
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: run_testsuite_CV
  tags:
    - crab3-shell
  script:
    - echo "Waiting for 20 minutes after task submission..."
    - sleep 1200 
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=CMSSW_13_0_2
    - export Client_Validation_Suite=true
    - ls workdir/submitted_tasks_CV
    - bash -x cicd/gitlab/CV_config.sh
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - workdir/submitted_tasks_CV
      policy: pull
    - key: submitted_tasks_CV_latest
      paths:
        - workdir/submitted_tasks_CV
      policy: pull

client_configuration_validation:
  rules:
    - if: $MANUAL_CI_PIPELINE_ID
      when: never
    - if: $CLIENT_CONFIGURATION_VALIDATION
    - if: $SKIP_DEPLOY || $FORCE_RELEASE
      when: never
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: run_testsuite_CCV
  tags:
    - crab3-shell
  script:
    # - echo "Waiting for 20 minutes after task submission..."
    # - sleep 1200 
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export CRABClient_version  # from .env
    - export REST_Instance  # from .env
    - export ROOT_DIR
    - export CMSSW_release=CMSSW_13_0_2
    - export Client_Configuration_Validation=true
    - ls workdir/submitted_tasks_CCV
    - cicd/gitlab/retry.sh bash -x cicd/gitlab/CCV_config.sh
  cache:
    - key: $CI_PIPELINE_ID
      paths:
        - workdir/submitted_tasks_CCV
      policy: pull
    - key: submitted_tasks_CCV_latest
      paths:
        - workdir/submitted_tasks_CCV
      policy: pull

check_test_result:
  rules:
    - if: '$CHECK_TEST_RESULT == "false"'
      when: never
    - !reference [.default_rules, skip_all_base]
    - !reference [.default_rules, default]
    - !reference [.default_rules, release]
  stage: check_testsuite
  tags:
    - crab3-shell
  script:
    - source .env
    - export X509_USER_PROXY=$(cicd/gitlab/credFile.sh $X509_USER_PROXY x509)
    - export REST_Instance  # from .env
    - export CRABClient_version  # from .env
    - export ROOT_DIR
    - export CMSSW_release=CMSSW_13_0_2
    - export SCRAM_ARCH=el8_amd64_gcc11
    - export Check_Publication_Status=Yes
    - echo ${MANUAL_CI_PIPELINE_ID}
    # manual task name
    - |
        if [[ -n "${MANUAL_TASKNAME:-}" ]]; then
            echo "${MANUAL_TASKNAME}" > workdir/submitted_tasks_TS
            echo "${MANUAL_TASKNAME}" > workdir/submitted_tasks_CV
            echo "${MANUAL_TASKNAME}" > workdir/submitted_tasks_CCV
        fi
    - cicd/gitlab/retry.sh bash -x cicd/gitlab/executeStatusTracking.sh
  cache:
    - key: $MANUAL_CI_PIPELINE_ID
      fallback_keys:
        - $CI_PIPELINE_ID
        - submitted_tasks_TS_latest
        - submitted_tasks_CCV_latest
        - submitted_tasks_CV_latest
      paths:
        - workdir
      policy: pull

# if test is pass, retag with `*-stable`
release_stable:
  rules:
    - !reference [.default_rules, release]
  stage: tagging_release
  image:
    name: registry.cern.ch/cmscrab/buildtools
    entrypoint: [""]
  variables:
    GIT_STRATEGY: none
  script:
    - crane auth login -u ${CMSCRAB_REGISTRY_USER} -p ${CMSCRAB_REGISTRY_PASSWORD} ${CMSCRAB_REGISTRY_URL}
    # rest
    - crane cp registry.cern.ch/cmscrab/crabserver:${IMAGE_TAG} registry.cern.ch/cmscrab/crabserver:${RELEASE_IMAGE_TAG}
    # tw
    - crane cp registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtaskworker:${RELEASE_IMAGE_TAG}
    # monit
    - crane cp registry.cern.ch/cmscrab/crabtwmonit:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtwmonit:${RELEASE_IMAGE_TAG}
    # filebeat
    - crane cp registry.cern.ch/cmscrab/crabtwfilebeat:${IMAGE_TAG} registry.cern.ch/cmscrab/crabtwfilebeat:${RELEASE_IMAGE_TAG}
    # spark
    - crane cp registry.cern.ch/cmscrab/crabspark:${IMAGE_TAG} registry.cern.ch/cmscrab/crabspark:${RELEASE_IMAGE_TAG}