ci.yml

name: Continuous Integration

on:
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]
  schedule:
    # run this every week at 12 noon on Monday
    - cron: '0 12 * * 1'

env:
  EXAMPLE_DATA_FILE_VERSION: v5
  # Setting this did not appear to work. Instead will need to
  # find/replace when changing it.
  # CONTAINER_IMAGE: ghcr.io/lsstdesc/txpipe:latest

jobs:
  # Run a download step first so that it is in
  # the cache, because all the other jobs will be running
  # at the same time and so might miss it. 
  Download_Data:
    runs-on: ubuntu-latest
    steps:
    - name: Cache example data
      id: cache-example
      uses: actions/cache@v4
      env:
        cache-name: cache-example-data
      with:
        path: example.tar.gz
        # update this when we change package contents and want
        # to force an update
        key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}

    - name: Download test data
      if: steps.cache-example.outputs.cache-hit != 'true'
      run: |
        wget -O example.tar.gz  "https://portal.nersc.gov/cfs/lsst/txpipe/data/example.tar.gz"


  Run_Installer:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-14]
        include:
          - os: ubuntu-latest
            INSTALL_DEPS: sudo apt-get update && sudo apt-get -y install wget
          - os: macos-14
            INSTALL_DEPS: echo nothing to do
    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        submodules: true

    - name: Install wget
      run: |
        ${{ matrix.INSTALL_DEPS }}

    - name: Cache conda environment
      id: cache-conda
      uses: actions/cache@v4
      env:
        cache-name: cache-conda-environment
      with:
        path: ./conda
        key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
        fail-on-cache-miss: false

    - name: Run installer
      if: steps.cache-conda.outputs.cache-hit != 'true'
      run: |
        ./bin/install.sh

  Unit_Tests:
    runs-on: ${{ matrix.os }}
    needs: Run_Installer
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-14]

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        submodules: true

    - name: Restore conda environment
      id: restore-conda
      uses: actions/cache@v4
      env:
        cache-name: cache-conda-environment
      with:
        path: ./conda
        key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
        fail-on-cache-miss: true

    - name: Test with pytest
      run: |
        source conda/bin/activate
        ceci --version
        pytest txpipe

  Pipelines:
    runs-on: ${{ matrix.os }}
    needs: [Run_Installer, Download_Data]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-14]
        pipeline: [metadetect, metacal, redmagic, lensfit, metadetect_source_only, mock_shear]
        include:
          - os: ubuntu-latest
            INSTALL_DEPS: echo nothing to do
          - os: macos-14
            INSTALL_DEPS: brew update-reset && brew install --cask basictex && eval "$(/usr/libexec/path_helper)" && pdflatex --version

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4
      with:
        submodules: true

    - name: Install dependencies
      run: |
        ${{ matrix.INSTALL_DEPS }}

    - name: Restore conda environment
      id: restore-conda
      uses: actions/cache@v4
      env:
        cache-name: cache-conda-environment
      with:
        path: ./conda
        key: ${{ matrix.os }}-conda-${{ hashFiles('bin/install.sh','environment*.yml' ) }}
        fail-on-cache-miss: true

    - name: Restore example data
      uses: actions/cache@v4
      with:
        path: example.tar.gz
        key: example-data-${{ env.EXAMPLE_DATA_FILE_VERSION }}
        fail-on-cache-miss: true

    - name: Extract test data
      run: |
        tar -zxvf example.tar.gz

    - name: Run pipeline
      run: |
        mkdir -p cache/workspaces
        source ./conda/bin/activate
        # Just for the metadetect source pipeline
        # we activate the debug mode with an env var
        if [ "${{ matrix.pipeline }}" == "metadetect_source_only" ]; then
          export TX_DASK_DEBUG=1
        fi
        ceci examples/${{ matrix.pipeline }}/pipeline.yml

        if [ "${{ matrix.pipeline }}" == "mock_shear" ]; then
          test -f data/example/outputs_mock_shear/binned_shear_catalog.hdf5
        else
          test -f data/example/outputs_${{ matrix.pipeline }}/shear_xi_plus.png
        fi

    - name: Show logs
      if: always()
      run: |
        # always run
        cat data/example/logs_${{ matrix.pipeline }}/*

    - name: Create output report
      if: matrix.os == 'macos-14'
      run: |
          source ./conda/bin/activate
          eval "$(/usr/libexec/path_helper)"
          python bin/make_output_report.py data/example/outputs_${{ matrix.pipeline }} ${{ matrix.pipeline }}-report.md
          pandoc ${{ matrix.pipeline }}-report.md -o ${{ matrix.pipeline }}-report.pdf

    - name: Upload output report
      if: matrix.os == 'macos-14'
      uses: actions/upload-artifact@v4
      with:
        name: ${{ matrix.pipeline }}-report
        path: ${{ matrix.pipeline }}-report.pdf