From 7555bc6ac07538fbe6305d99138c0e5305893e11 Mon Sep 17 00:00:00 2001 From: VinzentRisch Date: Wed, 27 Nov 2024 15:19:30 +0100 Subject: [PATCH] added CI from moshpit --- .github/workflows/black.yaml | 13 +++ .github/workflows/ci-dev.yaml | 33 +++++++ .github/workflows/ci.yaml | 126 ++++++++++++++++++++++++ .github/workflows/ci.yml | 55 ----------- .github/workflows/dependent-issues.yaml | 44 +++++++++ .github/workflows/join-release.yaml | 6 ++ .github/workflows/q2-ci.yaml | 42 ++++++++ .github/workflows/tag-release.yaml | 7 ++ .github/workflows/upload-coverage.yaml | 66 +++++++++++++ ci/recipe/meta.yaml | 37 +++++++ 10 files changed, 374 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/black.yaml create mode 100644 .github/workflows/ci-dev.yaml create mode 100644 .github/workflows/ci.yaml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/dependent-issues.yaml create mode 100644 .github/workflows/join-release.yaml create mode 100644 .github/workflows/q2-ci.yaml create mode 100644 .github/workflows/tag-release.yaml create mode 100644 .github/workflows/upload-coverage.yaml create mode 100644 ci/recipe/meta.yaml diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml new file mode 100644 index 0000000..639b5af --- /dev/null +++ b/.github/workflows/black.yaml @@ -0,0 +1,13 @@ +name: Black + +on: [push, pull_request] + +jobs: + black: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + with: + version: "23.12.1" + options: ". --check --extend-exclude '.*_version\\.py$' --verbose" diff --git a/.github/workflows/ci-dev.yaml b/.github/workflows/ci-dev.yaml new file mode 100644 index 0000000..86eda95 --- /dev/null +++ b/.github/workflows/ci-dev.yaml @@ -0,0 +1,33 @@ +name: CI +on: + pull_request: + branches: ["main"] + push: + branches: ["main"] + +jobs: + ci: + uses: qiime2/distributions/.github/workflows/lib-ci-dev.yaml@dev + with: + distro: tiny + additional-reports-path: ./coverage.xml + additional-reports-name: coverage + + coverage: + needs: [ci] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v3 + name: 'Fetch coverage from builds' + with: + name: ${{ needs.ci.outputs.additional-reports-name }} + path: ${{ needs.ci.outputs.additional-reports-path }} + + - uses: codecov/codecov-action@v4 + name: 'Upload coverage' + with: + fail_ci_if_error: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a3e7c47 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,126 @@ +name: Test +on: + pull_request: + branches: ["main"] + push: + branches: ["main"] +env: + DISTRO: tiny + +jobs: + test: + runs-on: ubuntu-latest + outputs: + latest-dev-tag: ${{ steps.fetch_latest_tags.outputs.latest-dev-tag }} + latest-stable-tag: ${{ steps.fetch_latest_tags.outputs.latest-stable-tag }} + commit-msg: ${{ steps.get-commit-msg.outputs.commit-msg }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Checkout utilities + uses: actions/checkout@v4 + with: + repository: bokulich-lab/utilities + path: utilities + + - name: Get last commit + id: get-commit-msg + run: | + echo "$(git log --pretty=%B -n 1 --skip 1)" + echo "commit-msg=$(git log --pretty=%B -n 1 --skip 1)" >> $GITHUB_OUTPUT + + - name: Install dependencies + run: python -m pip install requests yq + + - name: Fetch latest tags + id: fetch-tags + run: | + latest_tags=$(python utilities/ci/get-tags.py) + echo "$latest_tags" > tags.txt + echo "latest-dev-tag=$(grep 'latest-dev-tag' tags.txt | cut -d '=' -f 2)" >> $GITHUB_OUTPUT + echo "latest-stable-tag=$(grep 'latest-stable-tag' tags.txt | cut -d '=' -f 2)" >> $GITHUB_OUTPUT + + - name: Create conda yaml + id: create-conda-yaml + run: | + commit_msg="${{ steps.get-commit-msg.outputs.commit-msg }}" + if [[ "$commit_msg" == *"[stable]"* ]] || [[ "$commit_msg" == *"[prod]"* ]]; then + tag="${{ steps.fetch-tags.outputs.latest-stable-tag }}" + else + tag="${{ steps.fetch-tags.outputs.latest-dev-tag }}" + fi + bash utilities/ci/get-dependencies.sh "$DISTRO" $tag utilities/ci/repositories.yaml + cat environment.yml >> $GITHUB_STEP_SUMMARY + echo "qiime-deps=$(tr '\n' ' ' < repo-urls.txt | xargs)" >> $GITHUB_OUTPUT + + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: "3.10" + mamba-version: 1.5.10 + channels: conda-forge,defaults + channel-priority: true + activate-environment: conda-env + condarc-file: utilities/ci/condarc + # use-only-tar-bz2: true + + - name: Get date + id: get-date + run: echo "today=$(/bin/date -u '+%Y%m%d')" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache conda env + uses: actions/cache@v3 + with: + path: /usr/share/miniconda/envs + key: + conda-${{ runner.os }}--${{ runner.arch }}--${{ + steps.get-date.outputs.today }}-${{ + hashFiles('environment.yml') }}-${{ env.CACHE_NUMBER + }} + env: + # Increase this value to reset cache if environment.yml has not changed + CACHE_NUMBER: 0 + id: cache + + - name: Update environment + run: mamba env update -n conda-env -f environment.yml + if: steps.cache.outputs.cache-hit != 'true' + + - name: Install dev versions of QIIME repos + run: mamba run -n conda-env pip install ${{ steps.create-conda-yaml.outputs.qiime-deps }} + + - name: Update specific dependency, if requested + if: contains(${{ steps.get-commit-msg.outputs.commit-msg }}, '[add:') + run: | + commit_msg="${{ steps.get-commit-msg.outputs.commit-msg }}" + IFS=',' read -ra ADDR <<< "$commit_msg" + for i in "${ADDR[@]}"; do + pkg_name=$(echo "$i" | awk -F':' '/\[add/{print $2}') + commit_hash=$(echo "$i" | awk -F':' '/\[add/{print $3}' | awk -F']' '{print $1}') + if [[ $(yq '.repositories[].name' utilities/ci/repositories.yaml | grep -c $pkg_name) -eq 1 ]]; then + pkg_url=$(yq ".repositories[] | select(.name == \"$pkg_name\") | .url" utilities/ci/repositories.yaml | tr -d '"') + mamba run -n conda-env pip install "git+$pkg_url@$commit_hash" + fi + done + + - name: Install plugin + run: | + mamba run -n conda-env pip install . + mamba run -n conda-env qiime dev refresh-cache + + - name: Install dev dependencies + run: mamba run -n conda-env pip install pytest pytest-cov coverage parameterized + + - name: Run tests + id: test + run: mamba run -n conda-env make test-cov + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: steps.test.outcome == 'success' + with: + name: coverage + path: coverage.xml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index e8a464f..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Test and lint - -on: - pull_request: - branches: ["main"] - push: - branches: ["main"] - -jobs: - build-and-test-qiime2-tiny-2025-4: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, macos-13] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: q2-ms-qiime2-tiny-2025.4 - environment-file: environment-files/q2-ms-qiime2-tiny-2025.4.yml - auto-activate-base: false - - - name: Install plugin - shell: bash -l {0} - run: make install - - - name: Run tests - shell: bash -l {0} - run: make test - - lint: - runs-on: ubuntu-latest - steps: - - name: checkout source - uses: actions/checkout@v3 - - - name: set up python 3.11 - uses: actions/setup-python@v4 - with: - python-version: 3.11 - - - name: install dependencies - run: | - python -m pip install --upgrade pip - pip install -q https://github.com/qiime2/q2lint/archive/master.zip - pip install -q flake8 - - - name: run flake8 - run: flake8 - - - name: run q2lint - run: q2lint diff --git a/.github/workflows/dependent-issues.yaml b/.github/workflows/dependent-issues.yaml new file mode 100644 index 0000000..ea315b6 --- /dev/null +++ b/.github/workflows/dependent-issues.yaml @@ -0,0 +1,44 @@ +name: Dependent Issues + +on: + issues: + types: + - opened + - edited + - closed + - reopened + pull_request_target: + types: + - opened + - edited + - closed + - reopened + - synchronize + + schedule: + - cron: '0 0 * * *' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: z0al/dependent-issues@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_READ_TOKEN: ${{ secrets.GITHUB_READ_TOKEN }} + + with: + label: dependent + + # (Optional) Enable checking for dependencies in issues. + # Enable by setting the value to "on". Default "off" + check_issues: off + + ignore_dependabot: off + + keywords: depends on, blocked by, merge after + + comment: > + This PR/issue depends on: + + {{ dependencies }} diff --git a/.github/workflows/join-release.yaml b/.github/workflows/join-release.yaml new file mode 100644 index 0000000..298bce4 --- /dev/null +++ b/.github/workflows/join-release.yaml @@ -0,0 +1,6 @@ +name: join-release +on: + workflow_dispatch: {} +jobs: + release: + uses: qiime2/distributions/.github/workflows/lib-join-release.yaml@dev diff --git a/.github/workflows/q2-ci.yaml b/.github/workflows/q2-ci.yaml new file mode 100644 index 0000000..1858fae --- /dev/null +++ b/.github/workflows/q2-ci.yaml @@ -0,0 +1,42 @@ +name: "QIIME2: build and test" +on: issue_comment + +jobs: + react: + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/q2ci') }} + runs-on: ubuntu-latest + steps: + - uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ github.event.comment.id }} + body: | +
**Edit:** Running QIIME 2 CI... + reactions: "+1" + + ci: + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, '/q2ci') }} + uses: qiime2/distributions/.github/workflows/lib-ci-dev.yaml@dev + with: + distro: tiny + additional-reports-path: ./coverage.xml + additional-reports-name: coverage + + coverage: + needs: [ci] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v3 + name: 'Fetch coverage from builds' + with: + name: ${{ needs.ci.outputs.additional-reports-name }} + path: ${{ needs.ci.outputs.additional-reports-path }} + + - uses: codecov/codecov-action@v4 + name: 'Upload coverage' + with: + fail_ci_if_error: true + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/tag-release.yaml b/.github/workflows/tag-release.yaml new file mode 100644 index 0000000..fdbcab3 --- /dev/null +++ b/.github/workflows/tag-release.yaml @@ -0,0 +1,7 @@ +name: tag-release +on: + push: + branches: ["Release-*"] +jobs: + tag: + uses: qiime2/distributions/.github/workflows/lib-tag-release.yaml@dev diff --git a/.github/workflows/upload-coverage.yaml b/.github/workflows/upload-coverage.yaml new file mode 100644 index 0000000..019b40a --- /dev/null +++ b/.github/workflows/upload-coverage.yaml @@ -0,0 +1,66 @@ +name: Upload coverage report + +on: + workflow_run: + workflows: ["Test"] + types: + - completed + +jobs: + upload-coverage: + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + steps: + - name: 'Download artifact' + uses: actions/github-script@v7.0.1 + with: + script: | + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{github.event.workflow_run.id }}, + }); + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "coverage" + })[0]; + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{github.workspace}}/coverage.zip', Buffer.from(download.data)); + + - run: unzip coverage.zip + + - name: Find associated PR + id: pr + uses: actions/github-script@v7.0.1 + with: + script: | + const response = await github.rest.search.issuesAndPullRequests({ + q: 'repo:${{ github.repository }} is:pr sha:${{ github.event.workflow_run.head_sha }}', + per_page: 1, + }) + const items = response.data.items + if (items.length < 1) { + console.error('No PRs found') + return + } + const pullRequestNumber = items[0].number + console.info("Pull request number is", pullRequestNumber) + return pullRequestNumber + + - uses: codecov/codecov-action@v4 + name: Upload coverage report + with: + files: ./coverage.xml + fail_ci_if_error: true + override_pr: ${{ steps.pr.outputs.result }} + override_commit: ${{ github.event.workflow_run.head_sha }} + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/ci/recipe/meta.yaml b/ci/recipe/meta.yaml new file mode 100644 index 0000000..4b34627 --- /dev/null +++ b/ci/recipe/meta.yaml @@ -0,0 +1,37 @@ +{% set data = load_setup_py_data() %} +{% set version = data.get('version') or 'placehold' %} + +package: + name: q2-ms + version: {{ version }} + +source: + path: ../.. + +build: + script: make install + +requirements: + host: + - python {{ python }} + - setuptools + + run: + - qiime2 {{ qiime2_epoch }}.* + - q2-types {{ qiime2_epoch }}.* + - q2templates {{ qiime2_epoch }}.* + +test: + requires: + - coverage + - pytest-cov + imports: + - q2_ms + - qiime2.plugins.moshpit + commands: + - pytest --cov q2_ms --cov-report xml:coverage.xml --pyargs q2_ms + +about: + home: https://github.com/bokulich-lab/q2-ms + license: BSD-3-Clause + license_family: BSD