Skip to content
Continuous Integration (CI)

Cross obj data loading #987

Sign in to view logs

Cross obj data loading

Cross obj data loading #987

Workflow file for this run

.github/workflows/ci.yml at 0778e49
name: Continuous Integration (CI)
on:
push:
branches:
- main
pull_request:
branches:
- main
merge_group:
jobs:
python-test:
name: Format / Unit Tests / Python ${{ matrix.python-version }}
strategy:
fail-fast: false
matrix:
# Tests currently fail for 3.8 and 3.9.
python-version: ['3.10', '3.11']
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install library
run: sudo apt-get install -y libgraphviz-dev git-lfs && pip install .[dev]
- name: PyTest
run: make pytest
- name: PyLint
run: make pylint
# Pyflakes can see unused imports
- name: PyFlakes
run: make pyflakes
- name: Docstrings are defined
run: make flake8
notebook-test:
name: Notebook Tests / Python ${{ matrix.python-version }}
strategy:
fail-fast: false
matrix:
# Tests currently fail for 3.8 and 3.9.
python-version: ['3.10', '3.11']
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install library
run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]
- run: pip install ipython ipykernel nbconvert
# Notebooks are in the recipes/ folder.
- name: Run notebook
run: |
ipython kernel install --user --name croissant-notebook
for notebook in recipes/*ipynb
do
jupyter nbconvert \
--ExecutePreprocessor.timeout=600 \
--ExecutePreprocessor.kernel_name=croissant-notebook \
--to notebook \
--execute $notebook
done
python-format:
name: Python format
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: '3.11'
- uses: psf/black@stable
with:
options: --line-length 88 --preview --experimental-string-processing
src: './python/mlcroissant'
- uses: isort/isort-action@master
with:
sort-paths: './python/mlcroissant'
pytype-test:
name: PyType
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install dependencies except mlcroissant itself
run: >
pip install absl-py \
datasets \
etils[epath] \
GitPython \
jsonpath_rw \
mypy \
networkx \
pandas \
pyarrow \
pytest \
pytype \
rdflib \
requests \
tqdm
- name: PyType
run: make pytype
mypy-test:
name: MyPy
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Install dependencies except mlcroissant itself
run: >
pip install absl-py \
datasets \
etils[epath] \
GitPython \
jsonpath_rw \
mypy \
networkx \
pandas \
pyarrow \
pytest \
pytype \
rdflib \
requests \
tqdm
- name: MyPy
run: make mypy
validation-test:
name: Validation / JSON-LD Tests / Python 3.11
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install library
run: pip install .[git]
- name: Validate JSON-LD files
run: |
JSON_FILES=$(find ../../datasets/ -type f -name "*.json" ! -path '*/data/*')
for file in ${JSON_FILES}
do
echo "Validating ${file}..."
python mlcroissant/scripts/validate.py --file ${file}
done
generation-test:
name: Generation / JSON-LD Tests / Python 3.11
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./python/mlcroissant
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install library
run: pip install .
- name: Generate JSON-LD files - Titanic
run: mlcroissant load --file ../../datasets/titanic/metadata.json --record_set passengers
- name: Generate JSON-LD files - PASS
run: pip install .[image] && python mlcroissant/scripts/load.py --file ../../datasets/pass-mini/metadata.json --record_set images
wizard-python-test:
name: Wizard Python Test
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./wizard
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install library
run: pip install -r requirements.txt
- name: PyTest
run: pytest .
wizard-e2e-test:
name: Wizard End-to-End Tests
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./wizard
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install library
run: pip install -r requirements.txt
- name: Install mlcroissant
run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]
working-directory: ./python/mlcroissant
- uses: cypress-io/github-action@v6
with:
start: streamlit run app.py
wait-on: 'http://localhost:8501'
working-directory: ./wizard
- name: Upload screenshots
uses: actions/upload-artifact@v3
if: failure()
with:
name: cypress-screenshots
path: ./wizard/cypress/screenshots
retention-days: 1