ci.yml

name: Continuous Integration (CI)

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  merge_group:

jobs:
  python-test:
    name: Format / Unit Tests / Python ${{ matrix.python-version }}
    strategy:
      fail-fast: false
      matrix:
        # Tests currently fail for 3.8 and 3.9.
        python-version: ['3.10', '3.11']
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install library
      run: sudo apt-get install -y libgraphviz-dev git-lfs && pip install .[dev]

    - name: PyTest
      run: make pytest

    - name: PyLint
      run: make pylint

    # Pyflakes can see unused imports
    - name: PyFlakes
      run: make pyflakes

    - name: Docstrings are defined
      run: make flake8

  notebook-test:
    name: Notebook Tests / Python ${{ matrix.python-version }}
    strategy:
      fail-fast: false
      matrix:
        # Tests currently fail for 3.8 and 3.9.
        python-version: ['3.10', '3.11']
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install library
      run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]

    - run: pip install ipython ipykernel nbconvert
    
    # Notebooks are in the recipes/ folder. 
    - name: Run notebook
      run: |
        ipython kernel install --user --name croissant-notebook
        for notebook in recipes/*ipynb
          do
            jupyter nbconvert \
              --ExecutePreprocessor.timeout=600 \
              --ExecutePreprocessor.kernel_name=croissant-notebook \
              --to notebook \
              --execute $notebook
          done
          
  python-format:
    name: Python format
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - uses: actions/setup-python@v2
      with:
        python-version: '3.11'
    - uses: psf/black@stable
      with:
        options: --line-length 88 --preview --experimental-string-processing
        src: './python/mlcroissant'
    - uses: isort/isort-action@master
      with:
        sort-paths: './python/mlcroissant'

  pytype-test:
    name: PyType
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: Install dependencies except mlcroissant itself
      run: >
        pip install absl-py \
          datasets \
          etils[epath] \
          GitPython \
          jsonpath_rw \
          mypy \
          networkx \
          pandas \
          pyarrow \
          pytest \
          pytype \
          rdflib \
          requests \
          tqdm

    - name: PyType
      run: make pytype
  
  mypy-test:
    name: MyPy
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    
    - name: Install dependencies except mlcroissant itself
      run: >
          pip install absl-py \
            datasets \
            etils[epath] \
            GitPython \
            jsonpath_rw \
            mypy \
            networkx \
            pandas \
            pyarrow \
            pytest \
            pytype \
            rdflib \
            requests \
            tqdm
    
    - name: MyPy
      run: make mypy


  validation-test:
    name: Validation / JSON-LD Tests / Python 3.11
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'

    - name: Install library
      run: pip install .[git]

    - name: Validate JSON-LD files
      run: |
        JSON_FILES=$(find ../../datasets/ -type f -name "*.json" !  -path '*/data/*')
        for file in ${JSON_FILES}
        do
          echo "Validating ${file}..."
          python mlcroissant/scripts/validate.py --file ${file}
        done

  generation-test:
    name: Generation / JSON-LD Tests / Python 3.11
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./python/mlcroissant
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'

    - name: Install library
      run: pip install .

    - name: Generate JSON-LD files - Titanic
      run: mlcroissant load --file ../../datasets/titanic/metadata.json --record_set passengers

    - name: Generate JSON-LD files - PASS
      run: pip install .[image] && python mlcroissant/scripts/load.py --file ../../datasets/pass-mini/metadata.json --record_set images

  wizard-python-test:
    name: Wizard Python Test
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./wizard
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'

    - name: Install library
      run: pip install -r requirements.txt

    - name: PyTest
      run: pytest .

  wizard-e2e-test:
    name: Wizard End-to-End Tests
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./wizard
    steps:
    - uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'

    - name: Install library
      run: pip install -r requirements.txt

    - name: Install mlcroissant
      run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]
      working-directory: ./python/mlcroissant

    - uses: cypress-io/github-action@v6
      with:
        start: streamlit run app.py
        wait-on: 'http://localhost:8501'
        working-directory: ./wizard

    - name: Upload screenshots
      uses: actions/upload-artifact@v3
      if: failure()
      with:
          name: cypress-screenshots
          path: ./wizard/cypress/screenshots
          retention-days: 1