Skip to content

Commit

Permalink
Merge pull request #372 from LSSTDESC/ci-images
Browse files Browse the repository at this point in the history 
Updating CI
  • Loading branch information
@joezuntz
joezuntz authored Sep 20, 2024
2 parents cd99860 + 3ade84e commit eddd99d
Show file tree
Hide file tree
Showing 5 changed files with 197 additions and 241 deletions.
322 changes: 83 additions & 239 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,18 @@ on:

env:
EXAMPLE_DATA_FILE_VERSION: v3
# Setting this did not appear to work. Instead will need to
# find/replace when changing it.
# CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe-dev:latest


jobs:
# Run a download step first so that it is in
# the cache, because all the other jobs will be running
# at the same time and so might miss it. The other jobs
# still do try the download themselves in case something
# doesn't quite work
# at the same time and so might miss it.
Download_Data:
runs-on: ubuntu-latest
steps:
- name: Cache example data
id: cache-example
uses: actions/cache@v3
uses: actions/cache@v4
env:
cache-name: cache-example-data
with:
Expand All @@ -41,287 +37,135 @@ jobs:
wget -O example.tar.gz "https://portal.nersc.gov/cfs/lsst/txpipe/data/example.tar.gz"
Unit_Tests:
runs-on: ubuntu-latest

container:
image: ghcr.io/lsstdesc/txpipe:v1.0

steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true

- name: Test with pytest
run: |
ceci --version
pytest txpipe
Metacal_Pipeline:
runs-on: ubuntu-latest

needs: Download_Data

container:
image: ghcr.io/lsstdesc/txpipe:v1.0

steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true

- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metacal pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metacal/pipeline.yml
test -f data/example/outputs_metacal/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_metacal/*
Metadetect_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data

container:
image: ghcr.io/lsstdesc/txpipe:v1.0

Run_Installer:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14]
include:
- os: ubuntu-latest
INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
- os: macos-14
INSTALL_DEPS: echo nothing to do
steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
submodules: true

- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metadetect pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metadetect/pipeline.yml
test -f data/example/outputs_metadetect/shear_xi_plus.png
- name: Run randoms pipeline
run: |
ceci examples/randoms/pipeline_randomsonly.yml
test -f data/example/outputs_randoms/random_cats.hdf5
- name: Run notebook
run: |
jupyter nbconvert --to notebook --execute "notebooks/Welcome to TXPipe.ipynb"
- name: Show logs
if: ${{ always() }}
- name: Install wget
run: |
tail -n +1 data/example/logs_metadetect/*
Metadetect_Source_Only_Pipeline:
runs-on: ubuntu-latest
needs: Download_Data

container:
image: ghcr.io/lsstdesc/txpipe:v1.0
${{ matrix.INSTALL_DEPS }}
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true

- name: Restore example data
uses: actions/cache/restore@v3
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run metadetect pipeline
run: |
mkdir -p cache/workspaces
ceci examples/metadetect_source_only/pipeline.yml
test -f data/example/metadetect_source_only/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/metadetect_source_only/*
Lensfit_Pipeline:
runs-on: ubuntu-latest

needs: Download_Data

container:
image: ghcr.io/lsstdesc/txpipe:v1.0

steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true

- name: Restore example data
uses: actions/cache/restore@v3
- name: Cache conda environment
id: cache-conda
uses: actions/cache@v4
env:
cache-name: cache-conda-environment
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run lensfit pipeline
run: |
mkdir -p cache/workspaces
ceci examples/lensfit/pipeline.yml
test -f data/example/outputs_lensfit/shear_xi_plus.png
path: ./conda
key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
fail-on-cache-miss: false

- name: Show logs
if: ${{ always() }}
- name: Run installer
if: steps.cache-conda.outputs.cache-hit != 'true'
run: |
tail -n +1 data/example/logs_lensfit/*
./bin/install.sh
Redmagic_Pipeline:
runs-on: ubuntu-latest

needs: Download_Data

container:
image: ghcr.io/lsstdesc/txpipe:v1.0
Unit_Tests:
runs-on: ${{ matrix.os }}
needs: Run_Installer
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14]

steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
submodules: true

- name: Restore example data
uses: actions/cache/restore@v3
- name: Restore conda environment
id: restore-conda
uses: actions/cache@v4
env:
cache-name: cache-conda-environment
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
path: ./conda
key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Run redmagic pipeline
run: |
mkdir -p cache/workspaces
ceci examples/redmagic/pipeline.yml
test -f data/example/outputs_redmagic/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
run: |
tail -n +1 data/example/logs_redmagic/*
Other_Pipeline_Dry_Runs:
runs-on: ubuntu-latest

container:
image: ghcr.io/lsstdesc/txpipe:v1.0

steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
submodules: true
- name: Dry-run pipelines
- name: Test with pytest
run: |
ceci --dry-run examples/2.2i/pipeline.yml
ceci --dry-run examples/buzzard/pipeline.yml
ceci --dry-run examples/cosmodc2/pipeline.yml
ceci --dry-run examples/skysim/pipeline.yml
source conda/bin/activate
ceci --version
pytest txpipe
Test_Auto_Installer:
Pipelines:
runs-on: ${{ matrix.os }}
needs: Download_Data
needs: [Run_Installer, Download_Data]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-14]
pipeline: [metadetect, metacal, redmagic, lensfit, metadetect_source_only]
include:
- os: ubuntu-latest
INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
INSTALL_DEPS: echo nothing to do
- os: macos-14
INSTALL_DEPS: brew update-reset && brew install wget
INSTALL_DEPS: brew update-reset && brew install --cask basictex && eval "$(/usr/libexec/path_helper)" && pdflatex --version

steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v4
with:
submodules: true

- name: Install wget
run: ${{ matrix.INSTALL_DEPS }}
- name: Install dependencies
run: |
${{ matrix.INSTALL_DEPS }}
- name: Run Auto-Install
run: ./bin/install.sh
- name: Restore conda environment
id: restore-conda
uses: actions/cache@v4
env:
cache-name: cache-conda-environment
with:
path: ./conda
key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
fail-on-cache-miss: true

- name: Restore example data
uses: actions/cache/restore@v3
uses: actions/cache@v4
with:
path: example.tar.gz
# update this when we change package contents and want
# to force an update
key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
fail-on-cache-miss: true

- name: Extract test data
run: |
tar -zxvf example.tar.gz
- name: Test after auto-install
- name: Run pipeline
run: |
mkdir -p cache/workspaces
source ./conda/bin/activate
ceci examples/metadetect/pipeline.yml
ceci examples/${{ matrix.pipeline }}/pipeline.yml
test -f data/example/outputs_${{ matrix.pipeline }}/shear_xi_plus.png
- name: Show logs
if: ${{ always() }}
- name: Create output report
if: matrix.os == 'macos-14'
run: |
tail -n +1 data/example/logs_metadetect/*
source ./conda/bin/activate
eval "$(/usr/libexec/path_helper)"
python bin/make_output_report.py data/example/outputs_${{ matrix.pipeline }} ${{ matrix.pipeline }}-report.md
pandoc ${{ matrix.pipeline }}-report.md -o ${{ matrix.pipeline }}-report.pdf
- name: Upload output report
if: matrix.os == 'macos-14'
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.pipeline }}-report
path: ${{ matrix.pipeline }}-report.pdf
Loading

0 comments on commit eddd99d

Please sign in to comment.