From fd65ca4b94e3025327c62a4b83105487d8f35565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20Backenk=C3=B6hler?= Date: Mon, 13 Nov 2023 14:23:09 +0100 Subject: [PATCH] CI: try parallelizing notebook execution --- .github/workflows/ci.yml | 236 ++++++++++++++++++++++----------------- 1 file changed, 136 insertions(+), 100 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29b5149f..3aa4d2ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,126 +8,162 @@ on: branches: - "master" - "dev" + workflow_dispatch: schedule: # every two weeks on Monday at 3 am - cron: "0 3/336 * * 1" jobs: - test: - name: Pytest - runs-on: ${{ matrix.cfg.os }} - strategy: - fail-fast: false - matrix: - cfg: - - os: ubuntu-latest - python-version: "3.8" - - os: ubuntu-latest - python-version: "3.9" - - os: macos-latest - python-version: "3.8" - - os: windows-latest - python-version: "3.8" - - env: - PYVER: ${{ matrix.cfg.python-version }} - PACKAGE: teachopencadd + setup: + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - name: Checkout Repository + uses: actions/checkout@v3 + - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 with: - python-version: ${{ matrix.cfg.python-version }} miniforge-variant: Mambaforge miniforge-version: latest activate-environment: teachopencadd environment-file: devtools/test_env.yml - - name: Additional info about the build - shell: bash - run: | - uname -a - df -h - ulimit -a - - - name: Install package - shell: bash -l {0} - run: | - pip install . --no-deps -vv + test: + needs: setup + runs-on: ubuntu-latest - - name: Environment Information - shell: bash -l {0} - run: | - conda info --all - conda list + strategy: + matrix: + notebook: + - teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb + - teachopencadd/talktorials/T002_compound_adme/talktorial.ipynb + - teachopencadd/talktorials/T003_compound_unwanted_substructures/talktorial.ipynb - - name: Test CLI - shell: bash -l {0} - run: | - teachopencadd -h - pytest -v --cov=${PACKAGE} --cov-report=xml --color=yes ${PACKAGE}/tests/ + steps: + - name: Checkout Repository + uses: actions/checkout@v3 - name: Run tests shell: bash -l {0} run: | - PYTEST_ARGS="--nbval-lax --current-env --dist loadscope --numprocesses 2" - - PYTEST_IGNORE_T001="--ignore=teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb" - PYTEST_IGNORE_T018="--ignore=teachopencadd/talktorials/T018_automated_cadd_pipeline/talktorial.ipynb" - PYTEST_IGNORE_T019="--ignore=teachopencadd/talktorials/T019_md_simulation/talktorial.ipynb" - PYTEST_IGNORE_T020="--ignore=teachopencadd/talktorials/T020_md_analysis/talktorial.ipynb" - PYTEST_IGNORE_T035="--ignore=teachopencadd/talktorials/T035_graph_neural_networks/talktorial.ipynb" - PYTEST_IGNORE_T036="--ignore=teachopencadd/talktorials/T036_e3_equivariant_gnn/talktorial.ipynb" - PYTEST_IGNORE_T038="--ignore=teachopencadd/talktorials/T038_protein_ligand_interaction_prediction/talktorial.ipynb" - - IGNORE="$PYTEST_IGNORE_T001" - - if [ "$RUNNER_OS" == "Windows" ]; then - # Ignore T019 under Windows, see https://github.com/volkamerlab/teachopencadd/issues/313 - IGNORE="$IGNORE $PYTEST_IGNORE_T019" - IGNORE="$IGNORE $PYTEST_IGNORE_T035" - IGNORE="$IGNORE $PYTEST_IGNORE_T036" - IGNORE="$IGNORE $PYTEST_IGNORE_T038" - fi - - if [ "$RUNNER_OS" == "Linux" ]; then - IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385 - fi - - if [ "$RUNNER_OS" == "macOS" ]; then - IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385 - IGNORE="$IGNORE $PYTEST_IGNORE_T019" - IGNORE="$IGNORE $PYTEST_IGNORE_T035" # may cause kernel dying in pytest - IGNORE="$IGNORE $PYTEST_IGNORE_T036" # may cause kernel dying in pytest - fi - pytest $PYTEST_ARGS teachopencadd/talktorials/ $IGNORE - format: - name: Black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - name: Install format dependencies - run: python -m pip install black-nb shyaml - - name: Run black-nb - run: | - black-nb -l 99 --check teachopencadd/talktorials/T*/talktorial.ipynb - - readmes: - name: READMEs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - name: Install nbformat - run: python -m pip install nbformat shyaml - - name: Check READMEs are up-to-date - run: | - for path in teachopencadd/talktorials/T*/talktorial.ipynb; do - python devtools/regenerate_readmes.py --output README.md $path - done - # If the script above introduced changes (it shouldn't!), git diff will have output and an error code - git diff --exit-code + #test: + #name: Pytest + #runs-on: ${{ matrix.cfg.os }} + #strategy: + #fail-fast: false + #matrix: + #cfg: + #- os: ubuntu-latest + #python-version: "3.8" + #- os: ubuntu-latest + #python-version: "3.9" + #- os: macos-latest + #python-version: "3.8" + #- os: windows-latest + #python-version: "3.8" + + #env: + #PYVER: ${{ matrix.cfg.python-version }} + #PACKAGE: teachopencadd + + #steps: + #- uses: actions/checkout@v3 + + #- uses: conda-incubator/setup-miniconda@v2 + #with: + #python-version: ${{ matrix.cfg.python-version }} + #miniforge-variant: Mambaforge + #miniforge-version: latest + #activate-environment: teachopencadd + #environment-file: devtools/test_env.yml + + #- name: Additional info about the build + #shell: bash + #run: | + #uname -a + #df -h + #ulimit -a + + #- name: Install package + #shell: bash -l {0} + #run: | + #pip install . --no-deps -vv + + #- name: Environment Information + #shell: bash -l {0} + #run: | + #conda info --all + #conda list + + #- name: Test CLI + #shell: bash -l {0} + #run: | + #teachopencadd -h + #pytest -v --cov=${PACKAGE} --cov-report=xml --color=yes ${PACKAGE}/tests/ + + #- name: Run tests + #shell: bash -l {0} + #run: | + #PYTEST_ARGS="--nbval-lax --current-env --dist loadscope --numprocesses 2" + + #PYTEST_IGNORE_T001="--ignore=teachopencadd/talktorials/T001_query_chembl/talktorial.ipynb" + #PYTEST_IGNORE_T018="--ignore=teachopencadd/talktorials/T018_automated_cadd_pipeline/talktorial.ipynb" + #PYTEST_IGNORE_T019="--ignore=teachopencadd/talktorials/T019_md_simulation/talktorial.ipynb" + #PYTEST_IGNORE_T020="--ignore=teachopencadd/talktorials/T020_md_analysis/talktorial.ipynb" + #PYTEST_IGNORE_T035="--ignore=teachopencadd/talktorials/T035_graph_neural_networks/talktorial.ipynb" + #PYTEST_IGNORE_T036="--ignore=teachopencadd/talktorials/T036_e3_equivariant_gnn/talktorial.ipynb" + #PYTEST_IGNORE_T038="--ignore=teachopencadd/talktorials/T038_protein_ligand_interaction_prediction/talktorial.ipynb" + + #IGNORE="$PYTEST_IGNORE_T001" + + #if [ "$RUNNER_OS" == "Windows" ]; then + ## Ignore T019 under Windows, see https://github.com/volkamerlab/teachopencadd/issues/313 + #IGNORE="$IGNORE $PYTEST_IGNORE_T019" + #IGNORE="$IGNORE $PYTEST_IGNORE_T035" + #IGNORE="$IGNORE $PYTEST_IGNORE_T036" + #IGNORE="$IGNORE $PYTEST_IGNORE_T038" + #fi + + #if [ "$RUNNER_OS" == "Linux" ]; then + #IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385 + #fi + + #if [ "$RUNNER_OS" == "macOS" ]; then + #IGNORE="$IGNORE $PYTEST_IGNORE_T018" # see 385 + #IGNORE="$IGNORE $PYTEST_IGNORE_T019" + #IGNORE="$IGNORE $PYTEST_IGNORE_T035" # may cause kernel dying in pytest + #IGNORE="$IGNORE $PYTEST_IGNORE_T036" # may cause kernel dying in pytest + #fi + + #pytest $PYTEST_ARGS teachopencadd/talktorials/ $IGNORE + + #format: + #name: Black + #runs-on: ubuntu-latest + #steps: + #- uses: actions/checkout@v3 + #- uses: actions/setup-python@v3 + #- name: Install format dependencies + #run: python -m pip install black-nb shyaml + #- name: Run black-nb + #run: | + #black-nb -l 99 --check teachopencadd/talktorials/T*/talktorial.ipynb + + #readmes: + #name: READMEs + #runs-on: ubuntu-latest + #steps: + #- uses: actions/checkout@v3 + #- uses: actions/setup-python@v3 + #- name: Install nbformat + #run: python -m pip install nbformat shyaml + #- name: Check READMEs are up-to-date + #run: | + #for path in teachopencadd/talktorials/T*/talktorial.ipynb; do + #python devtools/regenerate_readmes.py --output README.md $path + #done + ## If the script above introduced changes (it shouldn't!), git diff will have output and an error code + #git diff --exit-code