Skip to content

[BUG] Open FlyteFile from remote path #14588

[BUG] Open FlyteFile from remote path

[BUG] Open FlyteFile from remote path #14588

Workflow file for this run

name: Build
# Schedule runs to run twice a day
on:
push:
branches:
- master
- 'release-v**'
pull_request:
schedule:
- cron: "0 13 * * *" # This schedule runs at 1pm UTC every day
env:
FLYTE_SDK_LOGGING_LEVEL: 10 # debug
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
detect-python-versions:
runs-on: ubuntu-latest
outputs:
python-versions: ${{ env.python_versions }}
steps:
- name: Set Python versions for run
run: |
if [[ ${{ github.event_name }} == "schedule" ]]; then
echo "python_versions=[\"3.9\",\"3.10\",\"3.11\",\"3.12\"]" >> $GITHUB_ENV
else
echo "python_versions=[\"3.9\", \"3.12\"]" >> $GITHUB_ENV
fi
build:
needs:
- detect-python-versions
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
steps:
- uses: actions/checkout@v4
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
run: |
pip install uv
make setup-global-uv
uv pip uninstall --system pandas pyarrow
uv pip freeze
- name: Test with coverage
run: |
make unit_test_codecov
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
files: coverage.xml
build-with-extras:
needs:
- detect-python-versions
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
steps:
- uses: actions/checkout@v4
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
run: |
pip install uv
make setup-global-uv
uv pip uninstall --system pandas pyarrow
uv pip freeze
- name: Run extras unit tests with coverage
# Skip this step if running on python 3.12 due to https://github.com/tensorflow/tensorflow/issues/62003
# and https://github.com/pytorch/pytorch/issues/110436
if: ${{ matrix.python-version != '3.12' }}
run: |
make unit_test_extras_codecov
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
files: coverage.xml
build-with-pandas:
needs:
- detect-python-versions
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
pandas: ["pandas<2.0.0", "pandas>=2.0.0"]
numpy: ["numpy<2.0.0", "numpy>=2.0.0"]
exclude:
- numpy: "numpy>=2.0.0"
pandas: "pandas<2.0.0"
- numpy: "numpy<2.0.0"
pandas: "pandas>=2.0.0"
- numpy: "numpy>=2.0.0"
python-version: "3.8"
steps:
- uses: actions/checkout@v4
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
run: |
pip install uv
make setup-global-uv
uv pip install --system --force-reinstall "${{ matrix.pandas }}" "${{ matrix.numpy }}"
uv pip freeze
- name: Test with coverage
run: |
make unit_test_codecov
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
files: coverage.xml
test-serialization:
needs:
- detect-python-versions
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
run: |
pip install uv
make setup-global-uv
uv pip freeze
- name: Test with coverage
run: |
make test_serialization_codecov
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
files: coverage.xml
integration:
needs:
- detect-python-versions
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
makefile-cmd:
[integration_test_codecov, integration_test_lftransfers_codecov]
steps:
# As described in https://github.com/pypa/setuptools_scm/issues/414, SCM needs git history
# and tags to work.
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache # sandbox has disk pressure, so we need to clear the cache to get more disk space.
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.in', 'requirements.in')) }}
- name: Install dependencies
run: |
pip install uv
make setup-global-uv
uv pip freeze
- name: Install FlyteCTL
uses: unionai-oss/flytectl-setup-action@master
- name: Setup Flyte Sandbox
run: |
flytectl demo start
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: |
image=moby/buildkit:master
network=host
- name: Build and push to local registry
uses: docker/build-push-action@v5
with:
context: .
file: Dockerfile.dev
build-args: |
PYTHON_VERSION=${{ matrix.python-version }}
PSEUDO_VERSION=1.999.0dev0
push: true
tags: localhost:30000/flytekit:dev
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Integration Test with coverage
env:
FLYTEKIT_IMAGE: localhost:30000/flytekit:dev
FLYTEKIT_CI: 1
PYTEST_OPTS: -n2
AWS_ENDPOINT_URL: 'http://localhost:30002'
AWS_ACCESS_KEY_ID: minio
AWS_SECRET_ACCESS_KEY: miniostorage
run: |
make ${{ matrix.makefile-cmd }}
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
build-plugins:
needs:
- build
- detect-python-versions
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ${{fromJson(needs.detect-python-versions.outputs.python-versions)}}
plugin-names:
# Please maintain an alphabetical order in the following list
- flytekit-airflow
- flytekit-async-fsspec
- flytekit-aws-athena
- flytekit-aws-batch
- flytekit-aws-sagemaker
- flytekit-bigquery
- flytekit-comet-ml
- flytekit-dask
- flytekit-data-fsspec
- flytekit-dbt
- flytekit-deck-standard
# TODO: remove dolt plugin - https://github.com/flyteorg/flyte/issues/5350
# flytekit-dolt
- flytekit-duckdb
- flytekit-envd
- flytekit-flyteinteractive
- flytekit-greatexpectations
- flytekit-hive
- flytekit-huggingface
- flytekit-identity-aware-proxy
- flytekit-inference
- flytekit-k8s-pod
- flytekit-kf-mpi
- flytekit-kf-pytorch
- flytekit-kf-tensorflow
- flytekit-memray
- flytekit-mlflow
- flytekit-mmcloud
- flytekit-modin
- flytekit-neptune
- flytekit-onnx-pytorch
- flytekit-onnx-scikitlearn
# onnx-tensorflow needs a version of tensorflow that does not work with protobuf>4.
# The issue is being tracked on the tensorflow side in https://github.com/tensorflow/tensorflow/issues/53234#issuecomment-1330111693
# flytekit-onnx-tensorflow
- flytekit-omegaconf
- flytekit-openai
- flytekit-pandera
- flytekit-papermill
- flytekit-polars
- flytekit-ray
- flytekit-snowflake
- flytekit-spark
- flytekit-sqlalchemy
- flytekit-vaex
- flytekit-whylogs
exclude:
- python-version: 3.9
plugin-names: "flytekit-aws-sagemaker"
# flytekit-modin depends on ray which does not have a 3.11 wheel yet.
# Issue tracked in https://github.com/ray-project/ray/issues/27881
- python-version: 3.11
plugin-names: "flytekit-modin"
- python-version: 3.11
plugin-names: "flytekit-ray"
# Great-expectations does not support python 3.11 due to sqlachemy>=2.0.0
# not being supported yet:
# https://github.com/great-expectations/great_expectations/issues/7020
- python-version: 3.11
plugin-names: "flytekit-greatexpectations"
# onnxruntime does not support python 3.10 yet
# https://github.com/microsoft/onnxruntime/issues/9782
- python-version: 3.11
plugin-names: "flytekit-onnx-pytorch"
- python-version: 3.11
plugin-names: "flytekit-onnx-scikitlearn"
- python-version: 3.11
plugin-names: "flytekit-onnx-tensorflow"
# vaex currently doesn't support python 3.11
- python-version: 3.11
plugin-names: "flytekit-vaex"
# whylogs does support python 3.11 dataclass restrictions
# See: https://github.com/flyteorg/flytekit/actions/runs/4493746408/jobs/7905368664
- python-version: 3.11
plugin-names: "flytekit-whylogs"
# apache-beam, one of flytekit-airflow dependencies, does not support python 3.12: https://github.com/apache/beam/issues/29149
- python-version: 3.12
plugin-names: "flytekit-airflow"
# Tensorflow is a dependency of flytekit-mlflow tests and that is not supported yet: https://github.com/tensorflow/tensorflow/issues/62003
- python-version: 3.12
plugin-names: "flytekit-mlflow"
# modin[ray] is a dependency needed to run the tests, unfortunately this is not supported in python 3.12 yet
- python-version: 3.12
plugin-names: "flytekit-modin"
# vaex currently doesn't support python 3.12
- python-version: 3.12
plugin-names: "flytekit-vaex"
# Ray does not support python 3.12 yet: https://github.com/ray-project/ray/issues/40211
- python-version: 3.12
plugin-names: "flytekit-ray"
# Segmentation fault on python 3.12: https://github.com/flyteorg/flyte/issues/5020
- python-version: 3.12
plugin-names: "flytekit-kf-pytorch"
steps:
- uses: actions/checkout@v4
- name: "Clear action cache"
uses: ./.github/actions/clear-action-cache
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip
uses: actions/cache@v3
with:
# This path is specific to Ubuntu
path: ~/.cache/pip
# Look to see if there is a cache hit for the corresponding requirements files
key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }}
- name: Install dependencies
run: |
pip install uv
# TODO: double-check if checking out all tags solves the issue
export SETUPTOOLS_SCM_PRETEND_VERSION="3.0.0"
make setup-global-uv
cd plugins/${{ matrix.plugin-names }}
uv pip install --system .
if [ -f dev-requirements.in ]; then uv pip install --system -r dev-requirements.in; fi
# TODO: move to protobuf>=5. Github issue: https://github.com/flyteorg/flyte/issues/5448
uv pip install --system -U $GITHUB_WORKSPACE "protobuf<5" "git+https://github.com/flyteorg/flyte.git@master#subdirectory=flyteidl"
# TODO: remove this when numpy v2 in onnx has been resolved
if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-sqlalchemy" || ${{ matrix.plugin-names }} == "flytekit-pandera" ]]; then
uv pip install --system "numpy<2.0.0"
fi
uv pip freeze
- name: Test with coverage
run: |
cd plugins/${{ matrix.plugin-names }}
# onnx plugins does not support protobuf>4 yet (in fact it is tensorflow that
# does not support that yet). More details in https://github.com/onnx/onnx/issues/4239.
if [[ ${{ matrix.plugin-names }} == *"onnx"* || ${{ matrix.plugin-names }} == "flytekit-whylogs" || ${{ matrix.plugin-names }} == "flytekit-mlflow" ]]; then
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
else
coverage run -m pytest tests --cov=./ --cov-report=xml --cov-append
fi
- name: Codecov
uses: codecov/[email protected]
with:
fail_ci_if_error: false
lint:
runs-on: ubuntu-latest
steps:
- name: Fetch the code
uses: actions/checkout@v4
- name: Set up Python 3.12
uses: actions/setup-python@v4
with:
python-version: 3.12
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/dev-requirements.in') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install uv
make setup-global-uv
uv pip freeze
- name: Lint
run: |
make lint