Skip to content
Continuous Integration

feat: Add generate_partitioned_path helper #252

Sign in to view logs

feat: Add generate_partitioned_path helper

feat: Add generate_partitioned_path helper #252

Workflow file for this run

.github/workflows/ci.yml at a4ce39f
name: Continuous Integration
on:
pull_request:
branches: ["**"]
push:
branches: ["main"]
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
checks:
name: Check for code errors
runs-on: ubuntu-latest
strategy:
matrix:
# Glue Pythonshell supports only Python 3.9.
# Glue ETL supports only Python 3.10.
python-version: ["3.9", "3.10"]
steps:
- uses: actions/[email protected]
- name: Set up Python ${{ matrix.python-version }}
uses: actions/[email protected]
with:
python-version: ${{ matrix.python-version }}
- name: Install Poetry Action
uses: snok/[email protected]
- name: Set up Virtualenv Cache
uses: actions/[email protected]
with:
path: .venv
key: ${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
restore-keys: ${{ runner.os }}-python${{ matrix.python-version }}-
- name: Install Python Dependencies
run: |
poetry check --lock
poetry install
- name: Lint code using ruff
run: |
poetry run ruff format . --check
poetry run ruff check .
- name: Check type annotations using mypy
run: MYPYPATH=src poetry run mypy .
- name: Check imports between modules
run: poetry run lint-imports
- name: Check if requirements.txt is up-to-date
run: poetry export --with=dev --output docker/requirements.txt && git diff --exit-code
- name: Run tests that do not need the Glue container
run: poetry run pytest test/test_options.py
- name: Build python packages
run: poetry build
tests:
name: Run tests in Glue container
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- name: Diffset
id: diffset
uses: softprops/[email protected]
with:
base: main
glue_files: |
**/*.py
docker/*
pyproject.toml
- name: Pull official glue image
if: steps.diffset.outputs.glue_files
run: docker pull amazon/aws-glue-libs:glue_libs_4.0.0_image_01
- name: Build container
if: steps.diffset.outputs.glue_files
run: USER_ID=$(id -u) TARGET=coverage docker compose --file docker/docker-compose.yml build glue-utils
- name: Run tests in container
if: steps.diffset.outputs.glue_files
run: USER_ID=$(id -u) TARGET=coverage docker compose --file docker/docker-compose.yml run --rm glue-utils -c "pytest --cov=glue_utils --cov-report=term --cov-report=xml"
- name: SonarCloud Scan
if: steps.diffset.outputs.glue_files
uses: SonarSource/sonarcloud-github-action@master
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
trivy:
name: Check dependencies for vulnerabilities
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
- name: Run Trivy vulnerability scanner
uses: aquasecurity/[email protected]
with:
scan-type: "fs"
output: "trivy.txt"
- name: Publish Trivy output to Summary
run: |
{
echo "### Trivy"
echo "<details><summary>Click to expand</summary>"
echo ""
echo '```'
cat trivy.txt
echo '```'
echo "</details>"
} >> "$GITHUB_STEP_SUMMARY"
dockerfile:
name: Check Dockerfile
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
- name: Validate Dockerfile using hadolint
uses: hadolint/[email protected]
with:
dockerfile: docker/Dockerfile
checkmake:
name: Check Makefile
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
- name: Check Makefile for errors
uses: Uno-Takashi/checkmake-action@main
checkov:
name: Run Checkov
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write
steps:
- uses: actions/[email protected]
- name: Check code using Checkov
uses: bridgecrewio/checkov-action@master
with:
directory: .
quiet: true
output_format: cli,sarif
output_file_path: console,checkov.sarif
- name: Upload SARIF results
if: success() || failure()
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: checkov.sarif
gitleaks:
name: Check for secrets
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0
- uses: gitleaks/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}