[Issue # 295] Replace GroupRecordSetStart/GroupRecordSetEnd by ReadFields. #970

Workflow file for this run

	name: Continuous Integration (CI)

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main
	merge_group:

	jobs:
	python-test:
	name: Format / Unit Tests / Python ${{ matrix.python-version }}
	strategy:
	fail-fast: false
	matrix:
	# Tests currently fail for 3.8 and 3.9.
	python-version: ['3.10', '3.11']
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install library
	run: sudo apt-get install -y libgraphviz-dev git-lfs && pip install .[dev]

	- name: PyTest
	run: make pytest

	- name: PyLint
	run: make pylint

	# Pyflakes can see unused imports
	- name: PyFlakes
	run: make pyflakes

	- name: Docstrings are defined
	run: make flake8

	notebook-test:
	name: Notebook Tests / Python ${{ matrix.python-version }}
	strategy:
	fail-fast: false
	matrix:
	# Tests currently fail for 3.8 and 3.9.
	python-version: ['3.10', '3.11']
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install library
	run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]

	- run: pip install ipython ipykernel nbconvert

	# Notebooks are in the recipes/ folder.
	- name: Run notebook
	run: \|
	ipython kernel install --user --name croissant-notebook
	for notebook in recipes/*ipynb
	do
	jupyter nbconvert \
	--ExecutePreprocessor.timeout=600 \
	--ExecutePreprocessor.kernel_name=croissant-notebook \
	--to notebook \
	--execute $notebook
	done

	python-format:
	name: Python format
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	- uses: actions/setup-python@v2
	with:
	python-version: '3.11'
	- uses: psf/black@stable
	with:
	options: --line-length 88 --preview --experimental-string-processing
	src: './python/mlcroissant'
	- uses: isort/isort-action@master
	with:
	sort-paths: './python/mlcroissant'

	pytype-test:
	name: PyType
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Install dependencies except mlcroissant itself
	run: >
	pip install absl-py \
	datasets \
	etils[epath] \
	GitPython \
	jsonpath_rw \
	mypy \
	networkx \
	pandas \
	pyarrow \
	pytest \
	pytype \
	rdflib \
	requests \
	tqdm

	- name: PyType
	run: make pytype

	mypy-test:
	name: MyPy
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Install dependencies except mlcroissant itself
	run: >
	pip install absl-py \
	datasets \
	etils[epath] \
	GitPython \
	jsonpath_rw \
	mypy \
	networkx \
	pandas \
	pyarrow \
	pytest \
	pytype \
	rdflib \
	requests \
	tqdm

	- name: MyPy
	run: make mypy


	validation-test:
	name: Validation / JSON-LD Tests / Python 3.11
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install library
	run: pip install .[git]

	- name: Validate JSON-LD files
	run: \|
	JSON_FILES=$(find ../../datasets/ -type f -name ".json" ! -path '/data/*')
	for file in ${JSON_FILES}
	do
	echo "Validating ${file}..."
	python mlcroissant/scripts/validate.py --file ${file}
	done

	generation-test:
	name: Generation / JSON-LD Tests / Python 3.11
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./python/mlcroissant
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install library
	run: pip install .

	- name: Generate JSON-LD files - Titanic
	run: mlcroissant load --file ../../datasets/titanic/metadata.json --record_set passengers

	- name: Generate JSON-LD files - PASS
	run: pip install .[image] && python mlcroissant/scripts/load.py --file ../../datasets/pass-mini/metadata.json --record_set images

	wizard-python-test:
	name: Wizard Python Test
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./wizard
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install library
	run: pip install -r requirements.txt

	- name: PyTest
	run: pytest .

	wizard-e2e-test:
	name: Wizard End-to-End Tests
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: ./wizard
	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install library
	run: pip install -r requirements.txt

	- name: Install mlcroissant
	run: sudo apt-get install -y libgraphviz-dev && pip install .[dev]
	working-directory: ./python/mlcroissant

	- uses: cypress-io/github-action@v6
	with:
	start: streamlit run app.py
	wait-on: 'http://localhost:8501'
	working-directory: ./wizard

	- name: Upload screenshots
	uses: actions/upload-artifact@v3
	if: failure()
	with:
	name: cypress-screenshots
	path: ./wizard/cypress/screenshots
	retention-days: 1

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Issue # 295] Replace GroupRecordSetStart/GroupRecordSetEnd by ReadFields. #970

Workflow file

[Issue # 295] Replace GroupRecordSetStart/GroupRecordSetEnd by ReadFields. #970

Jobs

Run details

Workflow file for this run