CI: fixed build and test, polished them, prepare for publishing #32

Workflow file for this run

.github/workflows/package.yml at 911d508

	name: Package

	# This workflow handles:
	# 1. Code quality checks for both Rust and Python
	# 2. Building the main Python package
	# 3. Building Rust service extensions (memory, s3)
	# 4. Verifying package installation and imports
	# Future: PyPI publishing on tags

	on:
	push:
	branches: [ main ]
	paths-ignore:
	- '*/.md'
	- '*/.rst'
	- 'docs/**'
	pull_request:
	branches: [ main ]
	paths-ignore:
	- '*/.md'
	- '*/.rst'
	- 'docs/**'

	jobs:
	check-rust:
	name: Rust Checks
	runs-on: ubuntu-24.04
	timeout-minutes: 10
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache Rust dependencies
	uses: actions/cache@v3
	with:
	path: \|
	~/.cargo/registry
	~/.cargo/git
	target
	key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

	- name: Check format
	run: cargo fmt --all -- --check

	- name: Check clippy
	run: cargo clippy --all-targets --all-features -- -D warnings

	check-python:
	name: Python Checks
	runs-on: ubuntu-24.04
	timeout-minutes: 10
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Cache pip
	uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}

	- name: Set up Python 3.11
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Install dev dependencies
	run: pip install -e ".[dev]"

	- name: Check format
	run: ruff format . --check

	- name: Check style
	run: ruff check .

	build-main:
	name: Build Main Package
	needs: [check-rust, check-python]
	runs-on: ubuntu-24.04
	timeout-minutes: 15
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set up Python 3.11
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Create dist directory
	run: mkdir -p dist

	- name: Build main package
	run: \|
	python -m pip install --upgrade pip build
	python -m build --outdir dist/

	- name: List dist contents
	run: ls -la dist/

	- name: Upload main artifact
	uses: actions/upload-artifact@v3
	with:
	name: dist-main
	path: dist/*
	if-no-files-found: error

	build-services:
	name: Build Service Packages
	needs: [build-main]
	runs-on: ubuntu-24.04
	timeout-minutes: 20
	strategy:
	fail-fast: false
	matrix:
	os: [ubuntu-24.04]
	target: [x86_64]
	service: [memory, s3]
	exclude:
	- os: ubuntu-24.04
	target: aarch64
	include:
	- os: ubuntu-24.04
	target: x86_64
	platform: manylinux2014

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set up Python 3.11
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Create dist directory
	run: mkdir -p dist

	- name: Build service package
	id: build-service
	uses: PyO3/maturin-action@v1
	with:
	target: ${{ matrix.target }}
	manylinux: ${{ matrix.platform }}
	container: quay.io/pypa/manylinux2014_x86_64
	args: --release --out dist --interpreter python3.11
	command: build
	working-directory: crates/service-${{ matrix.service }}

	- name: List dist contents
	run: \|
	echo "=== Service build output ==="
	ls -la dist/

	- name: Upload service artifact
	uses: actions/upload-artifact@v3
	with:
	name: dist-${{ matrix.os }}-${{ matrix.target }}-${{ matrix.service }}
	path: dist/*.whl
	if-no-files-found: error

	verify-install:
	name: Verify Installation
	needs: [build-services]
	runs-on: ubuntu-24.04
	timeout-minutes: 15
	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set up Python 3.11
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'

	- name: Download all artifacts
	uses: actions/download-artifact@v3
	with:
	path: dist

	- name: List downloaded artifacts
	run: \|
	echo "=== All artifacts ==="
	find dist -type f
	echo "=== Service artifacts ==="
	find dist/dist-ubuntu-24.04-x86_64-memory dist/dist-ubuntu-24.04-x86_64-s3 -type f

	- name: Prepare dist directory
	run: \|
	mkdir -p dist_combined
	# Copy service wheels first (only cp311 wheels)
	cp dist/dist-ubuntu-24.04-x86_64-memory/cp311.whl dist_combined/
	cp dist/dist-ubuntu-24.04-x86_64-s3/cp311.whl dist_combined/
	# Copy main package
	cp dist/dist-main/*.whl dist_combined/
	cp dist/dist-main/*.tar.gz dist_combined/
	echo "=== Contents of dist_combined ==="
	ls -la dist_combined/

	- name: Install packages in correct order
	run: \|
	python -m pip install --upgrade pip
	echo "=== Installing service packages ==="
	# Install memory service (only cp311 wheels)
	for whl in dist_combined/opendalfs_service_memorycp311.whl; do
	if [ -f "$whl" ]; then
	echo "Installing memory service: $whl"
	pip install "$whl"
	fi
	done
	# Install s3 service (only cp311 wheels)
	for whl in dist_combined/opendalfs_service_s3cp311.whl; do
	if [ -f "$whl" ]; then
	echo "Installing s3 service: $whl"
	pip install "$whl"
	fi
	done
	echo "=== Installing main package ==="
	pip install dist_combined/opendalfs-*.whl

	- name: Verify imports
	run: \|
	echo "=== Installed packages ==="
	pip list \| grep opendalfs
	echo "=== Verifying imports ==="
	python -c "import opendalfs"
	python -c "import opendalfs_service_memory"
	python -c "import opendalfs_service_s3"

	- name: Test full installation
	run: \|
	# Clean previous installation
	pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3
	# Install with all extras (use exact filename)
	echo "=== Installing with all extras ==="
	# First install service packages
	for whl in dist_combined/opendalfs_service_manylinux_2_17.whl; do
	if [ -f "$whl" ]; then
	echo "Installing service: $whl"
	pip install "$whl"
	fi
	done
	# Then install main package with all extras
	MAIN_WHEEL=$(find dist_combined -name "opendalfs-*.whl")
	echo "Installing main package with extras: $MAIN_WHEEL"
	pip install "$MAIN_WHEEL[all]"
	# Verify imports
	echo "=== Verifying imports ==="
	python -c "import opendalfs"
	python -c "import opendalfs_service_memory"
	python -c "import opendalfs_service_s3"

	- name: Cleanup
	if: always()
	run: \|
	pip uninstall -y opendalfs opendalfs_service_memory opendalfs_service_s3 \|\| true
	rm -rf dist dist_combined

	# PyPI publishing job (commented out for future use)
	# publish:
	# name: Publish to PyPI
	# needs: verify-install
	# runs-on: ubuntu-24.04
	# # Only run on tags
	# if: startsWith(github.ref, 'refs/tags/')
	#
	# steps:
	# - name: Download all artifacts
	# uses: actions/download-artifact@v3
	# with:
	# path: dist
	#
	# - name: Prepare dist directory
	# run: \|
	# mkdir -p dist_combined
	# cp -r dist// dist_combined/
	#
	# - name: Publish to PyPI
	# uses: pypa/gh-action-pypi-publish@release/v1
	# with:
	# password: ${{ secrets.PYPI_API_TOKEN }}
	# packages-dir: dist_combined/
	#
	# Required secrets for PyPI publishing:
	# - PYPI_API_TOKEN: API token from PyPI
	# Get it from: https://pypi.org/manage/account/token/
	# Add it to: https://github.com/fsspec/opendalfs/settings/secrets/actions

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CI: fixed build and test, polished them, prepare for publishing #32

Workflow file

CI: fixed build and test, polished them, prepare for publishing #32

Jobs

Run details

Workflow file for this run