Skip to content

Commit

Permalink
ci: Improvements to CI
Browse files Browse the repository at this point in the history
  • Loading branch information
dashmug committed Feb 20, 2024
1 parent 7c5cb63 commit dbf95ea
Show file tree
Hide file tree
Showing 7 changed files with 81 additions and 97 deletions.
31 changes: 24 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,11 @@ concurrency:

jobs:
python:
name: Python Checks
name: Python
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]

- name: Check shell scripts
uses: ludeeus/[email protected]
with:
scandir: '.'

- name: Setup Virtualenv Cache
uses: actions/[email protected]
with:
Expand Down Expand Up @@ -50,4 +45,26 @@ jobs:
poetry run ruff check .
- name: Check type annotations
run: poetry run mypy
run: poetry run mypy

- name: Check if requirements.txt is up to date
run: poetry export --output requirements.txt | diff requirements.txt -
shell:
name: Shell
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]

- name: Check shell scripts
uses: ludeeus/[email protected]
with:
scandir: '.'
dockerfile:
name: Dockerfile
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]

- name: Validate Dockerfile using hadolint
uses: hadolint/[email protected]

5 changes: 1 addition & 4 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@
"source.fixAll": "explicit"
}
},
"python.defaultInterpreterPath": "/usr/bin/python3",
"python.analysis.extraPaths": [
"/home/glue_user/aws-glue-libs/PyGlue.zip:/home/glue_user/spark/python/lib/py4j-0.10.9-src.zip:/home/glue_user/spark/python/"
],
"python.defaultInterpreterPath": "./.venv/bin/python",
"python.analysis.typeCheckingMode": "basic",
"python.analysis.diagnosticSeverityOverrides": {
"reportGeneralTypeIssues": "warning"
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
MAKEFLAGS += --no-print-directory

COMPOSE_RUN = USER_ID=$$(id -u) GROUP_ID=$$(id -g) docker compose run glue
COMPOSE_RUN = USER_ID=$$(id -u) docker compose run glue

COMPOSE_EXEC = USER_ID=$$(id -u) GROUP_ID=$$(id -g) docker compose exec glue
COMPOSE_EXEC = USER_ID=$$(id -u) docker compose exec glue


.PHONY: help
Expand Down Expand Up @@ -132,7 +132,7 @@ endif
.PHONY: clean-notebooks
clean-notebooks: ## Removes output cells from Jupyter notebooks
ifeq ($(PLATFORM), docker)
@jupyter nbconvert --clear-output notebooks/**.ipynb
@jupyter nbconvert --clear-output notebooks/**/*.ipynb
else
@$(COMPOSE_RUN) -c "make clean-notebooks"
endif
Expand Down
1 change: 0 additions & 1 deletion docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ services:
dockerfile: Dockerfile
args:
- USER_ID=${USER_ID:-}
- GROUP_ID=${GROUP_ID:-}
volumes:
- ~/.aws:/home/glue_user/.aws
- .:/home/glue_user/workspace
Expand Down
81 changes: 0 additions & 81 deletions notebooks/sample.ipynb

This file was deleted.

52 changes: 52 additions & 0 deletions notebooks/sample/notebook.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e77e0d17",
"metadata": {},
"source": [
"This script is taken from the [AWS Glue documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-libraries.html#develop-local-docker-image)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16d841a2-02e6-4f4a-9f4d-29d03f0cbf79",
"metadata": {},
"outputs": [],
"source": [
"from src.sample.script import GluePythonSampleTest\n",
"\n",
"\n",
"GluePythonSampleTest().run()\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4246de43-b703-4e36-8cc9-6fd2c22a8076",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "PySpark",
"language": "python",
"name": "pysparkkernel"
},
"language_info": {
"codemirror_mode": {
"name": "python",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "pyspark",
"pygments_lexer": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pyspark = "3.3.0"
line-length = 88
output-format = "full"
respect-gitignore = true
target-version = "py37"
target-version = "py310"

[tool.ruff.lint]
select = ["ALL"]
Expand Down

0 comments on commit dbf95ea

Please sign in to comment.