ci-build.yml

name: CI Build
on:
  workflow_dispatch:
  pull_request:
  # By default, the pull_request event type is not triggered when a PR is merged into main
  push:
    branches:
      - main
      - develop

jobs:
  lint-and-unit-test:
    name: Lint & Unit Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '16.x'
          cache: 'yarn'

      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          npm config set scripts-prepend-node-path true
          sudo apt-get install libgconf-2-4
          yarn --immutable

      # Linting and unit testing
      - name: Run linting
        run: yarn lint
      - name: Run unit tests
        run: yarn test:unit

      # Test coverage upload
      - name: Upload unit test coverage for the Common package
        if: success()
        uses: codecov/codecov-action@v3
        with:
          directory: ./packages/datagateway-common/
          flags: common
          fail_ci_if_error: true
      - name: Upload unit test coverage for the DataView package
        if: success()
        uses: codecov/codecov-action@v3
        with:
          directory: ./packages/datagateway-dataview/
          flags: dataview
          fail_ci_if_error: true
      - name: Upload unit test coverage for the Search package
        if: success()
        uses: codecov/codecov-action@v3
        with:
          directory: ./packages/datagateway-search/
          flags: search
          fail_ci_if_error: true
      - name: Upload unit test coverage for the Download package
        if: success()
        uses: codecov/codecov-action@v3
        with:
          directory: ./packages/datagateway-download/
          flags: download
          fail_ci_if_error: true

  dataview-e2e-tests:
    name: DataGateway DataView End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@v3
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      # Fixes on ICAT components needed for e2e tests
      - name: Add anon user to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install

      - name: Checkout datagateway-api
        uses: actions/checkout@v3
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v7.1.0

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '14.x'
      # Cache yarn dependencies/ restore the cached dependencies during future workflows
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "::set-output name=dir::$(yarn cache dir)"
      - name: Cache yarn dependencies
        uses: actions/cache@v3
        id: yarn-cache
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          npm config set scripts-prepend-node-path true
          sudo apt-get install libgconf-2-4
          yarn --frozen-lockfile

      - name: Run datagateway-dataview e2e tests
        run: yarn workspace datagateway-dataview run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: DataView-Screenshots
          path: packages/datagateway-dataview/cypress/screenshots

  download-e2e-tests:
    name: DataGateway Download End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@v3
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv

      # Fixes on ICAT components needed for e2e tests
      - name: Removing authenticator prefix for simple auth
        run: |
          sed -i 's/mechanism = simple/!mechanism = simple/' /home/runner/install/authn.simple/run.properties
      - name: Adding Chris481 user
        run: |
          sed -i '/user\.list/ s/$/ Chris481/' /home/runner/install/authn.simple/run.properties
      - name: Adding Chris481 user password
        run: |
          echo "user.Chris481.password = pw" >> /home/runner/install/authn.simple/run.properties
      - name: Reinstall authn.simple
        run: |
          cd /home/runner/install/authn.simple/ && ./setup -vv install
      - name: Add anon, root (simple without prefix) and Chris481 users to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= root Chris481 anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install
      - name: Disable Two Level Storage
        run: |
          sed -i '/plugin.archive.class/,/tidyBlockSize/ s/^/#/' /home/runner/install/ids.server/run.properties
      - name: Reinstall IDS Server
        run: |
          cd /home/runner/install/ids.server/ && python2 ./setup -vv install
      - name: Restart glassfish (ensure new ICAT rootUserNames is applied everywhere)
        run: asadmin restart-domain

      # Disable Globus for Download e2e tests
      - name: Login to ICAT
        run: |
          curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output
      - name: Disable Globus for testing
        run: |
          curl -k --request PUT 'https://localhost:8181/topcat/admin/downloadType/globus/status' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output` --data-urlencode 'facilityName=LILS' --data-urlencode 'disabled=True' --data-urlencode 'message="GLOBUS has been disabled for testing"'
      - name: Remove session ID data
        run: rm -f login_output

      - name: Checkout datagateway-api
        uses: actions/checkout@v3
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v7.1.0

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # DOI minter setup
      - name: Adding 'User-defined' DataPublicationType (needed for DOI minting api)
        run: cd datagateway-api/; poetry run python ../.github/add_doi_datapublicationtype.py

      - name: 'Add password to env file'
        run: echo DATACITE_PASSWORD=${{ secrets.DATACITE_PASSWORD }} >> ./.github/config.env

      - name: Run minting api
        run: docker run --env-file ./.github/config.env -p 8000:8000 --add-host host.docker.internal:host-gateway -d harbor.stfc.ac.uk/icat/doi-mint-api      

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '14.x'
      # Cache yarn dependencies/ restore the cached dependencies during future workflows
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "::set-output name=dir::$(yarn cache dir)"
      - name: Cache yarn dependencies
        uses: actions/cache@v3
        id: yarn-cache
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          npm config set scripts-prepend-node-path true
          sudo apt-get install libgconf-2-4
          yarn --frozen-lockfile

      - name: Run datagateway-download e2e tests
        run: yarn workspace datagateway-download run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: Download-Screenshots
          path: packages/datagateway-download/cypress/screenshots

  search-e2e-tests:
    name: DataGateway Search End to End Tests
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout repo
        uses: actions/checkout@v3
      - name: Add apt repo
        run: sudo add-apt-repository universe
      - name: Setup Java
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: 8
          java-package: jdk
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.6
          architecture: x64

      # ICAT Ansible clone and install dependencies
      - name: Checkout icat-ansible
        uses: actions/checkout@v3
        with:
          repository: icatproject-contrib/icat-ansible
          ref: master
          path: icat-ansible
      - name: Install Ansible
        run: pip install -r icat-ansible/requirements.txt

      # Prep for running the playbook
      - name: Create hosts file
        run: echo -e "[icat_test_hosts]\nlocalhost ansible_connection=local" > icat-ansible/hosts
      - name: Prepare vault pass
        run: echo -e "icattravispw" > icat-ansible/vault_pass.txt
      - name: Move vault to directory it'll get detected by Ansible
        run: mv icat-ansible/vault.yml icat-ansible/group_vars/all
      - name: Replace default payara user with Actions user
        run: |
          sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" icat-ansible/group_vars/all/vars.yml
      - name: Amending roles
        run: |
          sed -i 's/role: authn_uows_isis/role: authn_anon/' icat-ansible/icat_test_hosts.yml

      # Force hostname to localhost - bug fix for previous ICAT Ansible issues on Actions
      - name: Change hostname to localhost
        run: sudo hostname -b localhost

      # Remove existing MySQL installation so it doesn't interfere with GitHub Actions
      - name: Remove existing mysql
        run: |
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          sudo apt-get remove --purge "mysql*"
          sudo rm -rf /var/lib/mysql* /etc/mysql

      # Create local instance of ICAT
      - name: Run ICAT Ansible Playbook
        run: |
          ansible-playbook icat-ansible/icat_test_hosts.yml -i icat-ansible/hosts --vault-password-file icat-ansible/vault_pass.txt -vv
      - name: Add anon user to rootUserNames
        run: |
          awk -F" =" '/rootUserNames/{$2="= simple/root anon/anon";print;next}1' /home/runner/install/icat.server/run.properties > /home/runner/install/icat.server/run.properties.tmp
      - name: Apply rootUserNames change
        run: |
          mv -f /home/runner/install/icat.server/run.properties.tmp /home/runner/install/icat.server/run.properties
      - name: Reinstall ICAT Server
        run: |
          cd /home/runner/install/icat.server/ && ./setup -vv install

      - name: Checkout datagateway-api
        uses: actions/checkout@v3
        with:
          repository: ral-facilities/datagateway-api
          path: datagateway-api
          ref: v7.1.0

      # DataGateway API file setup
      - name: Create search_api_mapping.json
        run: cp datagateway-api/datagateway_api/search_api_mapping.json.example datagateway-api/datagateway_api/search_api_mapping.json
      - name: Create log file
        run: touch datagateway-api/datagateway_api/logs.log

      - name: Create config.yaml
        run: cp datagateway-api/datagateway_api/config.yaml.example datagateway-api/datagateway_api/config.yaml

      # DataGateway API dependencies
      - name: Install Poetry
        run: pip install poetry
      - name: Install dependencies
        run: cd datagateway-api/; poetry install
      - name: Add dummy data to icatdb
        run: |
          cd datagateway-api/; poetry run python -m util.icat_db_generator

      # Recreate Lucene indexes
      - name: Login to ICAT
        run: |
          curl -k --request POST 'https://localhost:8181/icat/session' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'json={"plugin":"simple", "credentials": [{"username":"root"}, {"password":"pw"}]}' > login_output
      - name: Recreate Datafile index
        run: |
          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Datafile/0' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
      - name: Recreate Dataset index
        run: |
          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Dataset/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
      - name: Recreate Investigation index
        run: |
          curl -k --request POST 'https://localhost:8181/icat/lucene/db/Investigation/1' --header 'Content-Type: application/x-www-form-urlencoded' --data-urlencode 'sessionId='`jq -r '.sessionId' login_output`
      - name: Let reindexes complete
        run: sleep 30
      - name: Remove session ID data
        run: rm -f login_output

      - name: Start API
        run: cd datagateway-api/; nohup poetry run python -m datagateway_api.src.main > api-output.txt &

      # E2E tests
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '14.x'
      # Cache yarn dependencies/ restore the cached dependencies during future workflows
      - name: Get yarn cache directory path
        id: yarn-cache-dir-path
        run: echo "::set-output name=dir::$(yarn cache dir)"
      - name: Cache yarn dependencies
        uses: actions/cache@v3
        id: yarn-cache
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-
      - name: Install dependencies
        # Ubuntu 16+ does not install libgconf-2-4 by default, so we need to install it ourselves (for Cypress)
        run: |
          npm config set scripts-prepend-node-path true
          sudo apt-get install libgconf-2-4
          yarn --frozen-lockfile

      - name: Run datagateway-search e2e tests
        run: yarn workspace datagateway-search run e2e
      - name: Upload Cypress screenshots
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: Search-Screenshots
          path: packages/datagateway-search/cypress/screenshots