From c1caaf7566d88030262cdec971a4c6ab4679dabf Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Wed, 3 Jan 2024 10:59:29 -0700 Subject: [PATCH 01/23] test pinned pendulum in CI --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index c158e97..9a81421 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,6 +55,7 @@ install_requires = pydantic~=1.10 sqlalchemy<2.0 # To address SQLAlchemy warning (https://sqlalche.me/e/b8d9) typing-extensions~=4.5 + pendulum==2.1.2 [options.packages.find] where = src From 27d7c7da1950ce6c9e1ef88c9e19b7680eba0bfe Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Wed, 3 Jan 2024 12:54:44 -0700 Subject: [PATCH 02/23] remove pendulum and lock step in tox --- setup.cfg | 1 - tox.ini | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 9a81421..c158e97 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,7 +55,6 @@ install_requires = pydantic~=1.10 sqlalchemy<2.0 # To address SQLAlchemy warning (https://sqlalche.me/e/b8d9) typing-extensions~=4.5 - pendulum==2.1.2 [options.packages.find] where = src diff --git a/tox.ini b/tox.ini index 0fa74ae..8a4d7c2 100644 --- a/tox.ini +++ b/tox.ini @@ -103,5 +103,4 @@ setenv = PIPENV_IGNORE_VIRTUALENVS = 1 deps = pipenv commands = - pipenv lock --dev - pipenv install --dev + pipenv install --dev --ignore-pipfile From 8111f4a7f91d22e00a1d42ffa1494eff328d0adb Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Thu, 4 Jan 2024 16:58:06 -0700 Subject: [PATCH 03/23] try pipenv tests --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6f503a..46b37ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,22 +38,22 @@ jobs: run: pipx run pre-commit run --all-files --show-diff-on-failure - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - - name: Build package distribution files - run: tox -e clean,build - - name: Record the paths of wheel and source tarball distributions - id: distribution-paths - run: | - echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT - echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT - - name: Store the distribution files for use in other stages - # `tests`, `pypi-publish`, and `docker-publish` will use the same - # pre-built distributions, so we make sure to release the exact - # same package that was tested - uses: actions/upload-artifact@v3 - with: - name: python-distribution-files - path: dist/ - retention-days: 1 + # - name: Build package distribution files + # run: tox -e clean,build + # - name: Record the paths of wheel and source tarball distributions + # id: distribution-paths + # run: | + # echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT + # echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT + # - name: Store the distribution files for use in other stages + # # `tests`, `pypi-publish`, and `docker-publish` will use the same + # # pre-built distributions, so we make sure to release the exact + # # same package that was tested + # uses: actions/upload-artifact@v3 + # with: + # name: python-distribution-files + # path: dist/ + # retention-days: 1 # Prevent scheduled workflow from ever being paused - name: Keepalive Workflow uses: gautamkrishnar/keepalive-workflow@1.1.0 @@ -80,21 +80,27 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - uses: actions/download-artifact@v3 - with: {name: python-distribution-files, path: dist/} - - name: Install tox-gh plugin - run: python -m pip install tox-gh>=1.2 - - name: Setup test suite - run: tox -vv --notest + # - uses: actions/download-artifact@v3 + # with: {name: python-distribution-files, path: dist/} + # - name: Install tox-gh plugin + # run: python -m pip install tox-gh>=1.2 + # - name: Setup test suite + # run: tox -vv --notest + - name: Install pipenv + run: python -m pip install pipenv + - name: Install dependencies + run: pipenv install -d - name: Run tests env: # New variables should match a `passenv` pattern under `[testenv]` in tox.ini SEVENBRIDGES_CONNECTION_URI: ${{ secrets.SEVENBRIDGES_CONNECTION_URI }} NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} - run: >- - tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' - -- -rFEx --durations 10 --color yes + run: + pipenv run pytest -vvv + # >- + # tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' + # -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From 3f869572a6e3b41b081a6c5f50858605cea905b6 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Thu, 4 Jan 2024 17:00:01 -0700 Subject: [PATCH 04/23] fix pipenv run line --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46b37ed..6a0de0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,8 +96,7 @@ jobs: SEVENBRIDGES_CONNECTION_URI: ${{ secrets.SEVENBRIDGES_CONNECTION_URI }} NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} - run: - pipenv run pytest -vvv + run: pipenv run pytest -vvv # >- # tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' # -- -rFEx --durations 10 --color yes From 61cbcbd5976989e70add5f610e2d79c595b2fead Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:07:29 -0700 Subject: [PATCH 05/23] updates gitignore for venvs --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a21ac9e..f3adf74 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ MANIFEST # Metaflow files .metaflow + +# Virtual Environment Files +/venv +/py-orca-venv-* From f34842fff361f0e756efd3ff3f0254d53e9c5c8d Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:07:48 -0700 Subject: [PATCH 06/23] adds airflow constraints file --- requirements-airflow.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 requirements-airflow.txt diff --git a/requirements-airflow.txt b/requirements-airflow.txt new file mode 100644 index 0000000..adefbcd --- /dev/null +++ b/requirements-airflow.txt @@ -0,0 +1,2 @@ +-c https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.10.txt +apache-airflow==2.7.2 From 3a4d3d6c2eebe5a51bc8b1f2f7d080e65fa60f86 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:08:07 -0700 Subject: [PATCH 07/23] adds bahs script for setting up dev environment --- dev_setup.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 dev_setup.sh diff --git a/dev_setup.sh b/dev_setup.sh new file mode 100755 index 0000000..5dfeaaf --- /dev/null +++ b/dev_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Usage: ./dev_setup.sh + +# Check if Python version argument is provided +if [ -z "$1" ]; then + echo "Please provide the Python version as an argument." + exit 1 +fi +# Check if Python version is supported +if [ "$1" != "3.10" ] && [ "$1" != "3.11" ]; then + echo "Invalid Python version. Please use 3.10 or 3.11." + exit 1 +fi +# Set up and activate a Python 3.11 virtual environment +python$1 -m venv py-orca-venv-$1 +source py-orca-venv-$1/bin/activate +# Upgrade pip to latest version +pip install --upgrade pip +# Install airflow with constraints +pip install -r requirements-airflow.txt +# Install py-orca +pip install -e '.[all,testing,dev]' From 1d2f5853274c56779ff0924c0066df719896da91 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:10:59 -0700 Subject: [PATCH 08/23] updates dev_setup --- dev_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_setup.sh b/dev_setup.sh index 5dfeaaf..8660e97 100755 --- a/dev_setup.sh +++ b/dev_setup.sh @@ -9,7 +9,7 @@ if [ -z "$1" ]; then fi # Check if Python version is supported if [ "$1" != "3.10" ] && [ "$1" != "3.11" ]; then - echo "Invalid Python version. Please use 3.10 or 3.11." + echo "Unsupported Python version. Please use 3.10 or 3.11." exit 1 fi # Set up and activate a Python 3.11 virtual environment From 886da8f2609df04986c8154cd510d2e2876b81ad Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:12:12 -0700 Subject: [PATCH 09/23] updates documentation to remove pipenv --- CONTRIBUTING.md | 7 +++---- README.md | 15 --------------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1f8f3a..9ea249b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,19 +101,18 @@ This often provides additional considerations and avoids unnecessary work. cd orca ``` -4. Install `pipx` to easily run Python CLI tools like `tox` and `pipenv`. - 4. Create an isolated virtual environment containing package dependencies, including those needed for development (*e.g.* testing, documentation) by running: ```console - pipx run tox -e pipenv + ./dev_setup.sh + source py-orca-venv-/bin/activate ``` 5. Install [pre-commit] hooks: ``` - pipenv run pre-commit install + pre-commit install ``` `orca` comes with a lot of hooks configured to automatically help the diff --git a/README.md b/README.md index d4c6648..f7e4282 100644 --- a/README.md +++ b/README.md @@ -24,21 +24,6 @@ Refer to [`.env.example`](.env.example) for the format of their values as well a Once your environment is set, you can create a virtual environment, install the Python dependencies, and run the demonstration script (after downloading it) as follows. Note that you will need to update the `s3_prefix` parameter so that it points to an S3 bucket that is accessible to your Tower workspace. -### Using pipenv -```bash -# Create and activate a Python virtual environment (tested with Python 3.11) -pipenv shell - -# Install Python dependencies -pipenv install - -# If development dependencies are required to be installed -pipenv install -d - -# Run the script using an example dataset -python3 demo.py run --dataset_id 'syn51514585' --s3_prefix 's3://orca-service-test-project-tower-bucket/outputs' -``` - ### Manually creating a virtual environment ```bash # Create and activate a Python virtual environment (tested with Python 3.10) From cab0ddccd51518a719137e8d13586ae293af7cf0 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:17:49 -0700 Subject: [PATCH 10/23] updates CI to test --- .github/workflows/ci.yml | 23 ++++++++++------------- tox.ini | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a0de0f..108675d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,26 +80,23 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - # - uses: actions/download-artifact@v3 - # with: {name: python-distribution-files, path: dist/} - # - name: Install tox-gh plugin - # run: python -m pip install tox-gh>=1.2 - # - name: Setup test suite - # run: tox -vv --notest - - name: Install pipenv - run: python -m pip install pipenv + - uses: actions/download-artifact@v3 + with: {name: python-distribution-files, path: dist/} + - name: Install tox-gh plugin + run: python -m pip install tox-gh>=1.2 + - name: Setup test suite + run: tox -vv --notest - name: Install dependencies - run: pipenv install -d + run: ./dev_setup.sh {{{ matrix.python }}} - name: Run tests env: # New variables should match a `passenv` pattern under `[testenv]` in tox.ini SEVENBRIDGES_CONNECTION_URI: ${{ secrets.SEVENBRIDGES_CONNECTION_URI }} NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} - run: pipenv run pytest -vvv - # >- - # tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' - # -- -rFEx --durations 10 --color yes + run: >- + tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' + -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/tox.ini b/tox.ini index 8a4d7c2..d775021 100644 --- a/tox.ini +++ b/tox.ini @@ -94,13 +94,13 @@ commands = python -m twine upload {posargs:--repository {env:TWINE_REPOSITORY:testpypi}} dist/* -[testenv:pipenv] -description = - Refresh the Pipfile.lock file based on the latest dependencies in setup.cfg -skip_install = True -changedir = {toxinidir} -setenv = - PIPENV_IGNORE_VIRTUALENVS = 1 -deps = pipenv -commands = - pipenv install --dev --ignore-pipfile +; [testenv:pipenv] +; description = +; Set up the python virtual environment and install dependencies using pip +; skip_install = True +; changedir = {toxinidir} +; setenv = +; PIPENV_IGNORE_VIRTUALENVS = 1 +; deps = pip +; commands = +; ./dev_setup.sh From 496bf74c2240084306339d92f06d3e816b611d00 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:24:26 -0700 Subject: [PATCH 11/23] updates ci.yml --- .github/workflows/ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 108675d..b5ba2df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,22 +38,22 @@ jobs: run: pipx run pre-commit run --all-files --show-diff-on-failure - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - # - name: Build package distribution files - # run: tox -e clean,build - # - name: Record the paths of wheel and source tarball distributions - # id: distribution-paths - # run: | - # echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT - # echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT - # - name: Store the distribution files for use in other stages - # # `tests`, `pypi-publish`, and `docker-publish` will use the same - # # pre-built distributions, so we make sure to release the exact - # # same package that was tested - # uses: actions/upload-artifact@v3 - # with: - # name: python-distribution-files - # path: dist/ - # retention-days: 1 + - name: Build package distribution files + run: tox -e clean,build + - name: Record the paths of wheel and source tarball distributions + id: distribution-paths + run: | + echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT + echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT + - name: Store the distribution files for use in other stages + # `tests`, `pypi-publish`, and `docker-publish` will use the same + # pre-built distributions, so we make sure to release the exact + # same package that was tested + uses: actions/upload-artifact@v3 + with: + name: python-distribution-files + path: dist/ + retention-days: 1 # Prevent scheduled workflow from ever being paused - name: Keepalive Workflow uses: gautamkrishnar/keepalive-workflow@1.1.0 From a2eb3bfa2727db4c6e5c1eaed731d1b6c03bfdfc Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 10:28:57 -0700 Subject: [PATCH 12/23] fix python version reference --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5ba2df..674538f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: - name: Setup test suite run: tox -vv --notest - name: Install dependencies - run: ./dev_setup.sh {{{ matrix.python }}} + run: ./dev_setup.sh ${{ matrix.python }} - name: Run tests env: # New variables should match a `passenv` pattern under `[testenv]` in tox.ini From 3d10a6072f939cd1ed18c8dd0100f4d0d059f9b6 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 11:44:09 -0700 Subject: [PATCH 13/23] add airflow requirements to tox installation --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index d775021..3cc7633 100644 --- a/tox.ini +++ b/tox.ini @@ -48,6 +48,7 @@ skip_install = True changedir = {toxinidir} deps = build: build[virtualenv] + -r {toxinidir}/requirements-airflow.txt passenv = SETUPTOOLS_* commands = From e5a93b56069445624677e1d68992d9ec09d6ed50 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 12:01:52 -0700 Subject: [PATCH 14/23] adds venv activation step, removes wheel path usage --- .github/workflows/ci.yml | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 674538f..ce97cfb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,9 @@ concurrency: jobs: prepare: runs-on: ubuntu-latest - outputs: - wheel-path: ${{ steps.distribution-paths.outputs.wheel }} - tarball-path: ${{ steps.distribution-paths.outputs.tarball }} + # outputs: + # wheel-path: ${{ steps.distribution-paths.outputs.wheel }} + # tarball-path: ${{ steps.distribution-paths.outputs.tarball }} steps: - uses: actions/checkout@v3 with: {fetch-depth: 0} # deep clone for setuptools-scm @@ -38,22 +38,22 @@ jobs: run: pipx run pre-commit run --all-files --show-diff-on-failure - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - - name: Build package distribution files - run: tox -e clean,build - - name: Record the paths of wheel and source tarball distributions - id: distribution-paths - run: | - echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT - echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT - - name: Store the distribution files for use in other stages - # `tests`, `pypi-publish`, and `docker-publish` will use the same - # pre-built distributions, so we make sure to release the exact - # same package that was tested - uses: actions/upload-artifact@v3 - with: - name: python-distribution-files - path: dist/ - retention-days: 1 + # - name: Build package distribution files + # run: tox -e clean,build + # - name: Record the paths of wheel and source tarball distributions + # id: distribution-paths + # run: | + # echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT + # echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT + # - name: Store the distribution files for use in other stages + # # `tests`, `pypi-publish`, and `docker-publish` will use the same + # # pre-built distributions, so we make sure to release the exact + # # same package that was tested + # uses: actions/upload-artifact@v3 + # with: + # name: python-distribution-files + # path: dist/ + # retention-days: 1 # Prevent scheduled workflow from ever being paused - name: Keepalive Workflow uses: gautamkrishnar/keepalive-workflow@1.1.0 @@ -88,6 +88,8 @@ jobs: run: tox -vv --notest - name: Install dependencies run: ./dev_setup.sh ${{ matrix.python }} + - name: activate venv + run: source py-orca-venv-${{ matrix.python }}/bin/activate - name: Run tests env: # New variables should match a `passenv` pattern under `[testenv]` in tox.ini @@ -95,8 +97,7 @@ jobs: NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} run: >- - tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' - -- -rFEx --durations 10 --color yes + tox -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From 4fc2cd2a05ea5f69a8aaffeaa3114675cd1bd418 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 12:05:27 -0700 Subject: [PATCH 15/23] removes artifact downloads for testing --- .github/workflows/ci.yml | 110 +++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce97cfb..9720635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,8 +80,8 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - - uses: actions/download-artifact@v3 - with: {name: python-distribution-files, path: dist/} + # - uses: actions/download-artifact@v3 + # with: {name: python-distribution-files, path: dist/} - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - name: Setup test suite @@ -109,57 +109,57 @@ jobs: env_vars: OS,PYTHON verbose: true - pypi-publish: - needs: [prepare, test] - if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 - with: {python-version: "3.11"} - - uses: actions/download-artifact@v3 - with: {name: python-distribution-files, path: dist/} - - name: Publish Python Package to PyPI - env: - TWINE_REPOSITORY: pypi - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: pipx run --spec 'tox!=3.0' tox -e publish + # pypi-publish: + # needs: [prepare, test] + # if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/setup-python@v4 + # with: {python-version: "3.11"} + # - uses: actions/download-artifact@v3 + # with: {name: python-distribution-files, path: dist/} + # - name: Publish Python Package to PyPI + # env: + # TWINE_REPOSITORY: pypi + # TWINE_USERNAME: __token__ + # TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + # run: pipx run --spec 'tox!=3.0' tox -e publish - docker-publish: - needs: [prepare, test] - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: {name: python-distribution-files, path: dist/} - - uses: docker/setup-qemu-action@v2 - - uses: docker/setup-buildx-action@v2 - - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - id: meta - uses: docker/metadata-action@v4 - with: - images: | - ghcr.io/${{ github.repository }} - tags: | - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - type=ref,event=branch - type=sha - - name: Publish Python Package to Docker Hub and GHCR - uses: docker/build-push-action@v3 - with: - platforms: linux/amd64,linux/arm64 - context: . - file: src/docker/Dockerfile - build-args: | - TARBALL_PATH=${{ needs.prepare.outputs.tarball-path }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + # docker-publish: + # needs: [prepare, test] + # if: ${{ github.event_name == 'push' }} + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + # - uses: actions/download-artifact@v3 + # with: {name: python-distribution-files, path: dist/} + # - uses: docker/setup-qemu-action@v2 + # - uses: docker/setup-buildx-action@v2 + # - uses: docker/login-action@v2 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ secrets.GITHUB_TOKEN }} + # - id: meta + # uses: docker/metadata-action@v4 + # with: + # images: | + # ghcr.io/${{ github.repository }} + # tags: | + # type=semver,pattern={{version}} + # type=semver,pattern={{major}}.{{minor}} + # type=semver,pattern={{major}} + # type=ref,event=branch + # type=sha + # - name: Publish Python Package to Docker Hub and GHCR + # uses: docker/build-push-action@v3 + # with: + # platforms: linux/amd64,linux/arm64 + # context: . + # file: src/docker/Dockerfile + # build-args: | + # TARBALL_PATH=${{ needs.prepare.outputs.tarball-path }} + # push: true + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} From c1b46e6b5be5ca9fa955e4fe65fe89e545fd591f Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 12:23:20 -0700 Subject: [PATCH 16/23] run tests without tox --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9720635..1a5a8d3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,8 +96,9 @@ jobs: SEVENBRIDGES_CONNECTION_URI: ${{ secrets.SEVENBRIDGES_CONNECTION_URI }} NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} - run: >- - tox -- -rFEx --durations 10 --color yes + run: pytest -vv + # >- + # tox -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From 1bbe64350f7137317f5cae6f8f5c90de27971475 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 12:53:02 -0700 Subject: [PATCH 17/23] adds requirements to testenv --- .github/workflows/ci.yml | 52 +++++++++++++++++++--------------------- tox.ini | 3 ++- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a5a8d3..2aadb09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,9 @@ concurrency: jobs: prepare: runs-on: ubuntu-latest - # outputs: - # wheel-path: ${{ steps.distribution-paths.outputs.wheel }} - # tarball-path: ${{ steps.distribution-paths.outputs.tarball }} + outputs: + wheel-path: ${{ steps.distribution-paths.outputs.wheel }} + tarball-path: ${{ steps.distribution-paths.outputs.tarball }} steps: - uses: actions/checkout@v3 with: {fetch-depth: 0} # deep clone for setuptools-scm @@ -38,22 +38,22 @@ jobs: run: pipx run pre-commit run --all-files --show-diff-on-failure - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - # - name: Build package distribution files - # run: tox -e clean,build - # - name: Record the paths of wheel and source tarball distributions - # id: distribution-paths - # run: | - # echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT - # echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT - # - name: Store the distribution files for use in other stages - # # `tests`, `pypi-publish`, and `docker-publish` will use the same - # # pre-built distributions, so we make sure to release the exact - # # same package that was tested - # uses: actions/upload-artifact@v3 - # with: - # name: python-distribution-files - # path: dist/ - # retention-days: 1 + - name: Build package distribution files + run: tox -e clean,build + - name: Record the paths of wheel and source tarball distributions + id: distribution-paths + run: | + echo "wheel=$(ls dist/*.whl)" >> $GITHUB_OUTPUT + echo "tarball=$(ls dist/*.tar.gz)" >> $GITHUB_OUTPUT + - name: Store the distribution files for use in other stages + # `tests`, `pypi-publish`, and `docker-publish` will use the same + # pre-built distributions, so we make sure to release the exact + # same package that was tested + uses: actions/upload-artifact@v3 + with: + name: python-distribution-files + path: dist/ + retention-days: 1 # Prevent scheduled workflow from ever being paused - name: Keepalive Workflow uses: gautamkrishnar/keepalive-workflow@1.1.0 @@ -80,25 +80,21 @@ jobs: - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - # - uses: actions/download-artifact@v3 - # with: {name: python-distribution-files, path: dist/} + - uses: actions/download-artifact@v3 + with: {name: python-distribution-files, path: dist/} - name: Install tox-gh plugin run: python -m pip install tox-gh>=1.2 - name: Setup test suite run: tox -vv --notest - - name: Install dependencies - run: ./dev_setup.sh ${{ matrix.python }} - - name: activate venv - run: source py-orca-venv-${{ matrix.python }}/bin/activate - name: Run tests env: # New variables should match a `passenv` pattern under `[testenv]` in tox.ini SEVENBRIDGES_CONNECTION_URI: ${{ secrets.SEVENBRIDGES_CONNECTION_URI }} NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} - run: pytest -vv - # >- - # tox -- -rFEx --durations 10 --color yes + run: >- + tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' + -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/tox.ini b/tox.ini index 3cc7633..bc4c28a 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,8 @@ passenv = HOME *_CONNECTION_URI SETUPTOOLS_* +deps = + -r {toxinidir}/requirements-airflow.txt extras = testing all @@ -48,7 +50,6 @@ skip_install = True changedir = {toxinidir} deps = build: build[virtualenv] - -r {toxinidir}/requirements-airflow.txt passenv = SETUPTOOLS_* commands = From 7e9ebcb57101028722c284df32941c57d61166b7 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 13:06:40 -0700 Subject: [PATCH 18/23] remove pipenv section from tox.ini --- tox.ini | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tox.ini b/tox.ini index bc4c28a..8102733 100644 --- a/tox.ini +++ b/tox.ini @@ -94,15 +94,3 @@ deps = twine commands = python -m twine check dist/* python -m twine upload {posargs:--repository {env:TWINE_REPOSITORY:testpypi}} dist/* - - -; [testenv:pipenv] -; description = -; Set up the python virtual environment and install dependencies using pip -; skip_install = True -; changedir = {toxinidir} -; setenv = -; PIPENV_IGNORE_VIRTUALENVS = 1 -; deps = pip -; commands = -; ./dev_setup.sh From eaff847d968f8738f3a9aa7d216004816605dfcd Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 13:06:52 -0700 Subject: [PATCH 19/23] uncomment publish sections --- .github/workflows/ci.yml | 106 +++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aadb09..ff37150 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,57 +106,57 @@ jobs: env_vars: OS,PYTHON verbose: true - # pypi-publish: - # needs: [prepare, test] - # if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - uses: actions/setup-python@v4 - # with: {python-version: "3.11"} - # - uses: actions/download-artifact@v3 - # with: {name: python-distribution-files, path: dist/} - # - name: Publish Python Package to PyPI - # env: - # TWINE_REPOSITORY: pypi - # TWINE_USERNAME: __token__ - # TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - # run: pipx run --spec 'tox!=3.0' tox -e publish + pypi-publish: + needs: [prepare, test] + if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: {python-version: "3.11"} + - uses: actions/download-artifact@v3 + with: {name: python-distribution-files, path: dist/} + - name: Publish Python Package to PyPI + env: + TWINE_REPOSITORY: pypi + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: pipx run --spec 'tox!=3.0' tox -e publish - # docker-publish: - # needs: [prepare, test] - # if: ${{ github.event_name == 'push' }} - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - uses: actions/download-artifact@v3 - # with: {name: python-distribution-files, path: dist/} - # - uses: docker/setup-qemu-action@v2 - # - uses: docker/setup-buildx-action@v2 - # - uses: docker/login-action@v2 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # - id: meta - # uses: docker/metadata-action@v4 - # with: - # images: | - # ghcr.io/${{ github.repository }} - # tags: | - # type=semver,pattern={{version}} - # type=semver,pattern={{major}}.{{minor}} - # type=semver,pattern={{major}} - # type=ref,event=branch - # type=sha - # - name: Publish Python Package to Docker Hub and GHCR - # uses: docker/build-push-action@v3 - # with: - # platforms: linux/amd64,linux/arm64 - # context: . - # file: src/docker/Dockerfile - # build-args: | - # TARBALL_PATH=${{ needs.prepare.outputs.tarball-path }} - # push: true - # tags: ${{ steps.meta.outputs.tags }} - # labels: ${{ steps.meta.outputs.labels }} + docker-publish: + needs: [prepare, test] + if: ${{ github.event_name == 'push' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/download-artifact@v3 + with: {name: python-distribution-files, path: dist/} + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: meta + uses: docker/metadata-action@v4 + with: + images: | + ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=sha + - name: Publish Python Package to Docker Hub and GHCR + uses: docker/build-push-action@v3 + with: + platforms: linux/amd64,linux/arm64 + context: . + file: src/docker/Dockerfile + build-args: | + TARBALL_PATH=${{ needs.prepare.outputs.tarball-path }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} From 6a78d3224fd7e1beb11ab86c6790b397b264dc49 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 13:15:02 -0700 Subject: [PATCH 20/23] format ci.yml --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff37150..a6f503a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,8 +93,8 @@ jobs: NEXTFLOWTOWER_CONNECTION_URI: ${{ secrets.NEXTFLOWTOWER_CONNECTION_URI }} SYNAPSE_CONNECTION_URI: ${{ secrets.SYNAPSE_CONNECTION_URI }} run: >- - tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' - -- -rFEx --durations 10 --color yes + tox --installpkg '${{ needs.prepare.outputs.wheel-path }}' + -- -rFEx --durations 10 --color yes - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: From 87ed86212e20fbea542a5fd33182bb3841616c17 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 13:22:11 -0700 Subject: [PATCH 21/23] updates docs --- .gitignore | 2 +- CONTRIBUTING.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index f3adf74..fc5bb87 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,6 @@ MANIFEST # Metaflow files .metaflow -# Virtual Environment Files +# Virtual Environment Directories /venv /py-orca-venv-* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ea249b..606f710 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,6 +109,8 @@ This often provides additional considerations and avoids unnecessary work. source py-orca-venv-/bin/activate ``` + **Note:** `py-orca` currently supports Python versions >=3.10 and <3.12. + 5. Install [pre-commit] hooks: ``` From ac14b309163c47a8933aa5a2391dac51d447e13f Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 13:29:22 -0700 Subject: [PATCH 22/23] specify python version in CONTRIBUTING --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 606f710..da3b47e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,11 +105,11 @@ This often provides additional considerations and avoids unnecessary work. including those needed for development (*e.g.* testing, documentation) by running: ```console - ./dev_setup.sh - source py-orca-venv-/bin/activate + ./dev_setup.sh 3.10 + source py-orca-venv-3.10/bin/activate ``` - **Note:** `py-orca` currently supports Python versions >=3.10 and <3.12. + **Note:** `py-orca` also supports Python version 3.11. 5. Install [pre-commit] hooks: From aa974be37cb332f9a9a8529ad6c3cdc4cb3dfa86 Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Mon, 8 Jan 2024 14:09:15 -0700 Subject: [PATCH 23/23] updates contributing --- CONTRIBUTING.md | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da3b47e..6a4685c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Welcome to `orca` contributor's guide. +Welcome to `py-orca` contributor's guide. This document focuses on getting any potential contributor familiarized with the development processes, but [other kinds of contributions] are also appreciated. @@ -16,7 +16,7 @@ behavior guidelines. ## Issue Reports -If you experience bugs or general issues with `orca`, please have a look +If you experience bugs or general issues with `py-orca`, please have a look on the [issue tracker]. If you don't see anything useful there, please feel free to fire an issue report. @@ -34,17 +34,17 @@ you help us to identify the root cause of the issue. ## Documentation Improvements -You can help improve `orca` docs by making them more readable and coherent, or +You can help improve py-`orca` docs by making them more readable and coherent, or by adding missing information and correcting mistakes. -`orca` documentation uses [Sphinx] as its main documentation compiler. +`py-orca` documentation uses [Sphinx] as its main documentation compiler. This means that the docs are kept in the same repository as the project code, and that any documentation update is done in the same way was a code contribution. The documentation is written using [CommonMark] with [MyST] extensions. :::{tip} Please notice that the [GitHub web interface] provides a quick way of - propose changes in `orca`'s files. While this mechanism can + propose changes in `py-orca`'s files. While this mechanism can be tricky for normal code contributions, it works perfectly fine for contributing to the docs, and can be quite handy. @@ -87,7 +87,7 @@ Before you work on any non-trivial code contribution it's best to first create a report in the [issue tracker] to start a discussion on the subject. This often provides additional considerations and avoids unnecessary work. -### Clone the repository +### Development Environment Setup 1. Create an user account on GitHub if you do not already have one. @@ -98,18 +98,20 @@ This often provides additional considerations and avoids unnecessary work. ```console git clone git@github.com:Sage-Bionetworks-Workflows/py-orca.git - cd orca + cd py-orca ``` 4. Create an isolated virtual environment containing package dependencies, including those needed for development (*e.g.* testing, documentation) by running: ```console - ./dev_setup.sh 3.10 - source py-orca-venv-3.10/bin/activate + ./dev_setup.sh + source py-orca-venv-/bin/activate ``` - **Note:** `py-orca` also supports Python version 3.11. + **Note:** + Ensure that you have a supported Python version installed on your local machine before running `dev_setup.sh`. `py-orca` currently supports Python version 3.10 and 3.11. You can [install](https://github.com/pyenv/pyenv?tab=readme-ov-file#installation) `pyenv`for easy Python version management. + 5. Install [pre-commit] hooks: