lint-and-test.yml

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
#
# OpenCRVS is also distributed under the terms of the Civil Registration
# & Healthcare Disclaimer located at http://opencrvs.org/license.
#
# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.

name: Lint and run unit tests

on:
  pull_request:
  push:
    branches:
      - develop

jobs:
  setup:
    name: Setup tests
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - uses: mskelton/changelog-reminder-action@v3
        # forked repos cannot access secrets.GITHUB_TOKEN which causes this step
        # to fail
        continue-on-error: true
        if: github.event_name == 'pull_request'
        with:
          message: >
            Oops! Looks like you forgot to update the changelog.
            When updating CHANGELOG.md, please consider the following:
              - Changelog is read by country implementors who might not always be familiar with all technical details of OpenCRVS. Keep language high-level, user friendly and avoid technical references to internals.
              - Answer "What's new?", "Why was the change made?" and "Why should I care?" for each change.
              - If it's a breaking change, include a migration guide answering "What do I need to do to upgrade?".

      - name: Get list of packages
        id: set-matrix
        run: |
          PACKAGES=$(ls -d packages/* | jq -R -s -c 'split("\n")[:-1]')
          echo "Found packages: $PACKAGES"
          echo "matrix=${PACKAGES}" >> $GITHUB_OUTPUT

  test:
    name: Test
    needs: setup
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        package: ${{fromJson(needs.setup.outputs.matrix)}}

    steps:
      - name: Checking out git repo
        uses: actions/checkout@v4

      - name: Check package.json and scripts
        id: check-scripts
        run: |
          if [ ! -f "${{ matrix.package }}/package.json" ]; then
            echo "No package.json found for ${{ matrix.package }}. Stopping pipeline."
            echo "skip=true" >> $GITHUB_OUTPUT
          else
            echo "skip=false" >> $GITHUB_OUTPUT

            if ! grep -q "\"test\":" "${{ matrix.package }}/package.json"; then
              echo "Test not found in ${{ matrix.package }}"
              echo "skip-test=true" >> $GITHUB_OUTPUT
            else
              echo "skip=false" >> $GITHUB_OUTPUT
            fi

            if ! grep -q "\"lint\":" "${{ matrix.package }}/package.json"; then
              echo "Lint scripts not found in ${{ matrix.package }}. Stopping pipeline."
              echo "skip-lint=true" >> $GITHUB_OUTPUT
            else
              echo "skip-lint=false" >> $GITHUB_OUTPUT
            fi
          fi

      - name: Use Node.js from .nvmrc
        if: steps.check-scripts.outputs.skip != 'true'
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Extract dependencies for ${{ matrix.package }}
        if: steps.check-scripts.outputs.skip != 'true'
        id: extract-dependencies
        run: |
          DEPENDENCIES=$(node -e "
            const { execSync } = require('child_process');
            const output = execSync('yarn --silent workspaces info', { encoding: 'utf-8' });
            const json = JSON.parse(output.replaceAll('@opencrvs', 'packages'));

            const getDependencies = (pkg) =>
              json[pkg].workspaceDependencies.concat(
                json[pkg].workspaceDependencies.flatMap(getDependencies)
              );

            console.log(
              getDependencies('${{ matrix.package }}').join(' ')
            );
          ")
          echo "DEPENDENCIES=${DEPENDENCIES}" >> $GITHUB_ENV
          echo "Found dependencies: $DEPENDENCIES"

      - name: Remove other package directories
        if: steps.check-scripts.outputs.skip != 'true'
        run: |
          for dir in packages/*; do
            if echo "${{ matrix.package }} $DEPENDENCIES" | grep -q -w "$dir"; then
              echo "Skipping $dir"
            else
              echo "Removing $dir"
              rm -rf "$dir"
            fi
          done

      - name: Cache Node.js dependencies
        uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/yarn/v6
          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}',matrix.package,'package.json')) }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Verify every file has a license header
        if: steps.check-scripts.outputs.skip != 'true'
        run: npx license-check-and-add check -f license-config.json

      - name: Runs dependency installation
        if: steps.check-scripts.outputs.skip != 'true'
        run: CI="" yarn install --frozen-lockfile

      - name: Build common package
        if: matrix.package != 'packages/client' && steps.check-scripts.outputs.skip != 'true' && contains(env.DEPENDENCIES, 'packages/commons')
        run: cd packages/commons && yarn build

      - name: Build components
        if: matrix.package != 'packages/client' && steps.check-scripts.outputs.skip != 'true' && contains(env.DEPENDENCIES, 'packages/components')
        run: |
          cd packages/components && yarn build

      # TODO: should run parallel to unit tests as can take as much as unit tests
      - name: Run linting
        if: matrix.package != 'packages/client' && steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-lint != 'true'
        run: cd ${{ matrix.package }} && yarn lint

      - name: Run Unit Test
        if: matrix.package != 'packages/client' && steps.check-scripts.outputs.skip != 'true' && steps.check-scripts.outputs.skip-test != 'true'
        run: cd ${{ matrix.package }} && yarn test

  prepare-client-tests:
    name: Prepare client tests
    runs-on: ubuntu-22.04
    steps:
      - name: Checking out git repo
        uses: actions/checkout@v4

      - name: Use Node.js from .nvmrc
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Extract dependencies for client
        id: extract-dependencies
        run: |
          DEPENDENCIES=$(node -e "
            const { execSync } = require('child_process');
            const output = execSync('yarn --silent workspaces info', { encoding: 'utf-8' });
            const json = JSON.parse(output.replaceAll('@opencrvs', 'packages'));

            const getDependencies = (pkg) =>
              json[pkg].workspaceDependencies.concat(
                json[pkg].workspaceDependencies.flatMap(getDependencies)
              );

            console.log(
              getDependencies('packages/client').join(' ')
            );
          ")
          echo "DEPENDENCIES=${DEPENDENCIES}" >> $GITHUB_ENV
          echo "Found dependencies: $DEPENDENCIES"

      - name: Remove other package directories
        run: |
          for dir in packages/*; do
            if echo "packages/client $DEPENDENCIES" | grep -q -w "$dir"; then
              echo "Skipping $dir"
            else
              echo "Removing $dir"
              rm -rf "$dir"
            fi
          done

      - name: Cache Node.js dependencies
        uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/yarn/v6
          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}','packages/client','package.json')) }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Install dependencies
        run: CI="" yarn install --frozen-lockfile

      - name: Build common package
        run: cd packages/commons && yarn build

      - name: Build components
        run: |
          cd packages/components && yarn build

      - name: Upload filesystem as artifact
        uses: actions/upload-artifact@v4.5.0
        with:
          name: client
          include-hidden-files: true
          path: |
            .
            !**/node_modules

  lint-client:
    name: Lint client
    needs: prepare-client-tests
    runs-on: ubuntu-22.04

    steps:
      - name: Download filesystem artifact
        uses: actions/download-artifact@v4.1.8
        with:
          name: client
          path: .

      - name: Use Node.js from .nvmrc
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Cache Node.js dependencies
        uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/yarn/v6
          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}','packages/client','package.json')) }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Install dependencies
        run: CI="" yarn install --frozen-lockfile

      - name: Compile
        run: cd packages/client && yarn test:compilation

      - name: Run linting
        run: cd packages/client && yarn lint

  run-visual-tests:
    name: Run visual tests
    needs: prepare-client-tests
    runs-on: ubuntu-22.04

    steps:
      - name: Download filesystem artifact
        uses: actions/download-artifact@v4.1.8
        with:
          name: client
          path: .

      - name: Use Node.js from .nvmrc
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Cache Node.js dependencies
        uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/yarn/v6
          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}','packages/client','package.json')) }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Install dependencies
        run: CI="" yarn install --frozen-lockfile

      # Without this, the fetch command will fail with
      # fatal: could not read Username for 'https://github.com': No such device or address
      - name: Remove extraheader config
        run: git config --unset http.https://github.com/.extraheader

      - name: Fetch full Git history
        run: git fetch --prune --unshallow
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Run Chromatic
        uses: chromaui/action@latest
        with:
          workingDir: packages/client
          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}

  test-client:
    name: Test client
    needs: prepare-client-tests
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        shards: [1, 2, 3, 4, 5]

    steps:
      - name: Download filesystem artifact
        uses: actions/download-artifact@v4.1.8
        with:
          name: client
          path: .

      - name: Use Node.js from .nvmrc
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Cache Node.js dependencies
        uses: actions/cache@v4
        with:
          path: |
            **/node_modules
            ~/.cache/yarn/v6
          key: node-${{ hashFiles('**/yarn.lock', format('{0}/{1}','packages/client','package.json')) }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Install dependencies
        run: CI="" yarn install --frozen-lockfile

      - name: Run Unit Test
        run: cd packages/client && yarn test -- --shard $(( ${{ strategy.job-index }} + 1 ))/${{ strategy.job-total }}

  lint-knip:
    name: Lint unused exports with Knip
    runs-on: ubuntu-22.04
    if: github.event_name == 'pull_request'
    steps:
      - name: Checkout base branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.base_ref }}
          path: base

      - name: Checkout the PR branch
        uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}
          path: pr

      - uses: actions/setup-node@v4

      - name: Install base dependencies
        run: yarn install --ignore-scripts
        working-directory: base

      - name: Install PR dependencies
        run: yarn install --ignore-scripts
        working-directory: pr

      - name: Run knip on base branch
        id: knip_base
        run: |
          npx knip --tags=-knipignore --no-exit-code --exports --reporter=markdown | sed -E 's/ +/ /g' | sed -E 's/:[0-9]+:[0-9]+//' > knip_report.md
          TOTAL=$(grep -oP '## [A-Za-z\s]+ \(\K[0-9]+' knip_report.md | awk '{sum+=$1} END {print sum}')
          echo "Total $TOTAL issue(s) on base branch."
          echo "total=${TOTAL}" >> $GITHUB_OUTPUT
        working-directory: base

      - name: Run knip on PR branch
        id: knip_pr
        run: |
          npx knip --tags=-knipignore --no-exit-code --exports --reporter=markdown | sed -E 's/ +/ /g' | sed -E 's/:[0-9]+:[0-9]+//' > knip_report.md
          TOTAL=$(grep -oP '## [A-Za-z\s]+ \(\K[0-9]+' knip_report.md | awk '{sum+=$1} END {print sum}')
          echo "Total $TOTAL issue(s) on PR branch."
          echo "total=${TOTAL}" >> $GITHUB_OUTPUT
        working-directory: pr

      - name: Compare base and PR totals
        if: ${{ steps.knip_pr.outputs.total > steps.knip_base.outputs.total }}
        run: |
          echo "## ⚠️ Total issues have increased in the PR branch." >> $GITHUB_STEP_SUMMARY
          echo "Differences:" >> $GITHUB_STEP_SUMMARY
          echo "\`\`\`diff" >> $GITHUB_STEP_SUMMARY
          diff base/knip_report.md pr/knip_report.md >> $GITHUB_STEP_SUMMARY
          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY

          exit 1

  build-toolkit:
    name: Build toolkit
    runs-on: ubuntu-22.04
    if: github.event_name == 'pull_request'
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Use Node.js from .nvmrc
        uses: actions/setup-node@v4
        with:
          node-version-file: .nvmrc

      - name: Install dependencies
        run: yarn install --frozen-lockfile

      - name: Build commons
        run: yarn build
        working-directory: packages/commons

      - name: Build events
        run: yarn build
        working-directory: packages/events

      - name: Build toolkit
        run: yarn build
        working-directory: packages/toolkit