Build

Add the Flyte agent to provision and manage K8s (data) service for deep learning (GNN) use cases #14580

Workflow file for this run

.github/workflows/pythonbuild.yml at b9c4dd1

	name: Build

	# Schedule runs to run twice a day

	on:
	push:
	branches:
	- master
	- 'release-v**'
	pull_request:
	schedule:
	- cron: "0 13 * * *" # This schedule runs at 1pm UTC every day

	env:
	FLYTE_SDK_LOGGING_LEVEL: 10 # debug

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
	cancel-in-progress: true

	jobs:
	detect-python-versions:
	runs-on: ubuntu-latest
	outputs:
	python-versions: ${{ env.python_versions }}
	steps:
	- name: Set Python versions for run
	run: \|
	if [[ ${{ github.event_name }} == "schedule" ]]; then
	echo "python_versions=[\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
	else
	echo "python_versions=[\"3.9\", \"3.12\"]" >> $GITHUB_ENV
	fi

	build:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: "Clear action cache"
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	pip install uv
	make setup-global-uv
	uv pip uninstall --system pandas pyarrow
	uv pip freeze
	- name: Test with coverage
	run: \|
	make unit_test_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	build-with-extras:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest, windows-latest, macos-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: "Clear action cache"
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	pip install uv
	make setup-global-uv
	uv pip uninstall --system pandas pyarrow
	uv pip freeze
	- name: Run extras unit tests with coverage
	# Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
	# and https://github.com/pytorch/pytorch/issues/110436
	if: ${{ matrix.python-version != '3.12' }}
	run: \|
	make unit_test_extras_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	build-with-pandas:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	pandas: ["pandas<2.0.0", "pandas>=2.0.0"]
	numpy: ["numpy<2.0.0", "numpy>=2.0.0"]
	exclude:
	- numpy: "numpy>=2.0.0"
	pandas: "pandas<2.0.0"
	- numpy: "numpy<2.0.0"
	pandas: "pandas>=2.0.0"
	- numpy: "numpy>=2.0.0"
	python-version: "3.8"

	steps:
	- uses: actions/checkout@v4
	- name: "Clear action cache"
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	pip install uv
	make setup-global-uv
	uv pip install --system --force-reinstall "${{ matrix.pandas }}" "${{ matrix.numpy }}"
	uv pip freeze
	- name: Test with coverage
	run: \|
	make unit_test_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	test-serialization:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	pip install uv
	make setup-global-uv
	uv pip freeze
	- name: Test with coverage
	run: \|
	make test_serialization_codecov
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	files: coverage.xml

	integration:
	needs:
	- detect-python-versions
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-latest]
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	makefile-cmd:
	[integration_test_codecov, integration_test_lftransfers_codecov]
	steps:
	# As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
	# and tags to work.
	- uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: "Clear action cache"
	uses: ./.github/actions/clear-action-cache # sandbox has disk pressure, so we need to clear the cache to get more disk space.
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
	- name: Install dependencies
	run: \|
	pip install uv
	make setup-global-uv
	uv pip freeze
	- name: Install FlyteCTL
	uses: unionai-oss/flytectl-setup-action@master
	- name: Setup Flyte Sandbox
	run: \|
	flytectl demo start
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	with:
	driver-opts: \|
	image=moby/buildkit:master
	network=host
	- name: Build and push to local registry
	uses: docker/build-push-action@v5
	with:
	context: .
	file: Dockerfile.dev
	build-args: \|
	PYTHON_VERSION=${{ matrix.python-version }}
	PSEUDO_VERSION=1.999.0dev0
	push: true
	tags: localhost:30000/flytekit:dev
	cache-from: type=gha
	cache-to: type=gha,mode=max
	- name: Integration Test with coverage
	env:
	FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
	FLYTEKIT_CI: 1
	PYTEST_OPTS: -n2
	AWS_ENDPOINT_URL: 'http://localhost:30002'
	AWS_ACCESS_KEY_ID: minio
	AWS_SECRET_ACCESS_KEY: miniostorage
	run: \|
	make ${{ matrix.makefile-cmd }}
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false

	build-plugins:
	needs:
	- build
	- detect-python-versions
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
	plugin-names:
	# Please maintain an alphabetical order in the following list
	- flytekit-airflow
	- flytekit-async-fsspec
	- flytekit-aws-athena
	- flytekit-aws-batch
	- flytekit-aws-sagemaker
	- flytekit-bigquery
	- flytekit-comet-ml
	- flytekit-dask
	- flytekit-data-fsspec
	- flytekit-dbt
	- flytekit-deck-standard
	# TODO: remove dolt plugin - https://github.com/flyteorg/flyte/issues/5350
	# flytekit-dolt
	- flytekit-duckdb
	- flytekit-envd
	- flytekit-flyteinteractive
	- flytekit-greatexpectations
	- flytekit-hive
	- flytekit-huggingface
	- flytekit-identity-aware-proxy
	- flytekit-inference
	- flytekit-k8sdataservice
	- flytekit-k8s-pod
	- flytekit-kf-mpi
	- flytekit-kf-pytorch
	- flytekit-kf-tensorflow
	- flytekit-memray
	- flytekit-mlflow
	- flytekit-mmcloud
	- flytekit-modin
	- flytekit-neptune
	- flytekit-onnx-pytorch
	- flytekit-onnx-scikitlearn
	# onnx-tensorflow needs a version of tensorflow that does not work with protobuf>4.
	# The issue is being tracked on the tensorflow side in https://github.com/tensorflow/tensorflow/issues/53234#issuecomment-1330111693
	# flytekit-onnx-tensorflow
	- flytekit-omegaconf
	- flytekit-openai
	- flytekit-pandera
	- flytekit-papermill
	- flytekit-polars
	- flytekit-ray
	- flytekit-snowflake
	- flytekit-spark
	- flytekit-sqlalchemy
	- flytekit-vaex
	- flytekit-whylogs
	exclude:
	- python-version: 3.9
	plugin-names: "flytekit-aws-sagemaker"
	# flytekit-modin depends on ray which does not have a 3.11 wheel yet.
	# Issue tracked in https://github.com/ray-project/ray/issues/27881
	- python-version: 3.11
	plugin-names: "flytekit-modin"
	- python-version: 3.11
	plugin-names: "flytekit-ray"
	# Great-expectations does not support python 3.11 due to sqlachemy>=2.0.0
	# not being supported yet:
	# https://github.com/great-expectations/great_expectations/issues/7020
	- python-version: 3.11
	plugin-names: "flytekit-greatexpectations"
	# onnxruntime does not support python 3.10 yet
	# https://github.com/microsoft/onnxruntime/issues/9782
	- python-version: 3.11
	plugin-names: "flytekit-onnx-pytorch"
	- python-version: 3.11
	plugin-names: "flytekit-onnx-scikitlearn"
	- python-version: 3.11
	plugin-names: "flytekit-onnx-tensorflow"
	# vaex currently doesn't support python 3.11
	- python-version: 3.11
	plugin-names: "flytekit-vaex"
	# whylogs does support python 3.11 dataclass restrictions
	# See: https://github.com/flyteorg/flytekit/actions/runs/4493746408/jobs/7905368664
	- python-version: 3.11
	plugin-names: "flytekit-whylogs"
	# apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
	- python-version: 3.12
	plugin-names: "flytekit-airflow"
	# Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
	- python-version: 3.12
	plugin-names: "flytekit-mlflow"
	# modin[ray] is a dependency needed to run the tests, unfortunately this is not supported in python 3.12 yet
	- python-version: 3.12
	plugin-names: "flytekit-modin"
	# vaex currently doesn't support python 3.12
	- python-version: 3.12
	plugin-names: "flytekit-vaex"
	# Ray does not support python 3.12 yet: https://github.com/ray-project/ray/issues/40211
	- python-version: 3.12
	plugin-names: "flytekit-ray"
	# Segmentation fault on python 3.12: https://github.com/flyteorg/flyte/issues/5020
	- python-version: 3.12
	plugin-names: "flytekit-kf-pytorch"
	steps:
	- uses: actions/checkout@v4
	- name: "Clear action cache"
	uses: ./.github/actions/clear-action-cache
	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}
	- name: Cache pip
	uses: actions/cache@v3
	with:
	# This path is specific to Ubuntu
	path: ~/.cache/pip
	# Look to see if there is a cache hit for the corresponding requirements files
	key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
	- name: Install dependencies
	run: \|
	pip install uv
	# TODO: double-check if checking out all tags solves the issue
	export SETUPTOOLS_SCM_PRETEND_VERSION="3.0.0"
	make setup-global-uv
	cd plugins/${{ matrix.plugin-names }}
	uv pip install --system .
	if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi
	# TODO: move to protobuf>=5. Github issue: https://github.com/flyteorg/flyte/issues/5448
	uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5" "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl"
	# TODO: remove this when numpy v2 in onnx has been resolved
	if [[ ${{ matrix.plugin-names }} == "onnx" \|\| ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" \|\| ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then
	uv pip install --system "numpy<2.0.0"
	fi
	uv pip freeze
	- name: Test with coverage
	run: \|
	cd plugins/${{ matrix.plugin-names }}
	# onnx plugins does not support protobuf>4 yet (in fact it is tensorflow that
	# does not support that yet). More details in https://github.com/onnx/onnx/issues/4239.
	if [[ ${{ matrix.plugin-names }} == "onnx" \|\| ${{ matrix.plugin-names }} == "flytekit-whylogs" \|\| ${{ matrix.plugin-names }} == "flytekit-mlflow" ]]; then
	PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
	else
	coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
	fi
	- name: Codecov
	uses: codecov/[email protected]
	with:
	fail_ci_if_error: false
	lint:
	runs-on: ubuntu-latest
	steps:
	- name: Fetch the code
	uses: actions/checkout@v4
	- name: Set up Python 3.12
	uses: actions/setup-python@v4
	with:
	python-version: 3.12
	- uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}
	restore-keys: \|
	${{ runner.os }}-pip-
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install uv
	make setup-global-uv
	uv pip freeze
	- name: Lint
	run: \|
	make lint

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add the Flyte agent to provision and manage K8s (data) service for deep learning (GNN) use cases #14580

Workflow file

Add the Flyte agent to provision and manage K8s (data) service for deep learning (GNN) use cases #14580

Jobs

Run details

Workflow file for this run