checks.yml

name: Checks

on:
  push:
    branches:
      - main
    paths:
      - '**' # Include all files by default
      - '!.devcontainer/**'
      - '!.vscode/**'
      - '!.git*'
      - '!*.md'
      - '!.github/**'
      - '.github/workflows/checks.yml' # Still include current workflow
  pull_request:
    branches:
      - main
    paths:
      - '**'
      - '!.devcontainer/**'
      - '!.vscode/**'
      - '!.git*'
      - '!*.md'
      - '!.github/**'
      - '.github/workflows/checks.yml'
  # Allow this workflow to be called from other workflows
  workflow_call:
    inputs:
      #  Requires at least one input to be valid, but in practice we don't need any
      dummy:
        type: string
        required: false

permissions:
  actions: write
  contents: write

jobs:
  checks:
    name: Checks
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
    steps:
      - uses: actions/checkout@v3
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "poetry"
      - name: Cache Models used with Tests
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/huggingface/hub/models--gpt2
            ~/.cache/huggingface/hub/models--NeelNanda--Attn_Only*
            ~/.cache/huggingface/hub/models--roneneldan--TinyStories-1M*
          key: ${{ runner.os }}-huggingface-models
      - name: Install dependencies
        run: |
          poetry lock --check
          poetry install --with dev
      - name: Check format
        run: make check-format
      - name: Unit test
        run: make unit-test
      - name: Docstring test
        run: make docstring-test
      # - name: Type check
      #   run: poetry run mypy transformer_lens
      - name: Build check
        run: poetry build
  
  # Acceptance tests are run in parallel with unit checks.
  acceptance-tests:
    name: Acceptance Tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
          cache: "poetry"
      - name: Cache Models used with Tests
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache/huggingface/hub/models--gpt2
            ~/.cache/huggingface/hub/models--NeelNanda--Attn_Only*
            ~/.cache/huggingface/hub/models--roneneldan--TinyStories-1M*
          key: ${{ runner.os }}-huggingface-models
      - name: Install dependencies
        run: |
          poetry lock --check
          poetry install --with dev
      - name: Acceptance test
        run: make acceptance-test

  notebook-checks:
    name: Notebook Checks
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
          cache: "poetry"
      - name: Re-use HuggingFace models cache
        uses: actions/cache/restore@v3
        with:
          path: ~/.cache/huggingface/hub
          key: ${{ runner.os }}-huggingface-models
      - name: Install dependencies
        run: |
          poetry lock --check
          poetry install --with dev,jupyter
      - name: Install pandoc
        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: pandoc
          version: 1.0
      - name: Check Notebook Output Consistency
        # Note: currently only checks notebooks we have specifically setup for this
        run: make notebook-test