ci: Improvements to CI

.github/workflows/ci.yml

@@ -12,16 +12,11 @@ concurrency:
 
 jobs:
   python:
-    name: Python Checks
+    name: Python
     runs-on: ubuntu-latest
     steps:
       - uses: actions/[email protected]
 
-      - name: Check shell scripts
-        uses: ludeeus/[email protected]
-        with:
-          scandir: '.'
-
       - name: Setup Virtualenv Cache
         uses: actions/[email protected]
         with:
@@ -50,4 +45,26 @@ jobs:
           poetry run ruff check .
 
       - name: Check type annotations
-        run: poetry run mypy
+        run: poetry run mypy
+
+      - name: Check if requirements.txt is up to date
+        run: poetry export --output requirements.txt | diff requirements.txt -
+  shell:
+    name: Shell
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Check shell scripts
+        uses: ludeeus/[email protected]
+        with:
+          scandir: '.'
+  dockerfile:
+    name: Dockerfile
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Validate Dockerfile using hadolint
+        uses: hadolint/[email protected]
+

.vscode/settings.json

@@ -14,10 +14,7 @@
       "source.fixAll": "explicit"
     }
   },
-  "python.defaultInterpreterPath": "/usr/bin/python3",
-  "python.analysis.extraPaths": [
-    "/home/glue_user/aws-glue-libs/PyGlue.zip:/home/glue_user/spark/python/lib/py4j-0.10.9-src.zip:/home/glue_user/spark/python/"
-  ],
+  "python.defaultInterpreterPath": "./.venv/bin/python",
   "python.analysis.typeCheckingMode": "basic",
   "python.analysis.diagnosticSeverityOverrides": {
     "reportGeneralTypeIssues": "warning"

Makefile

@@ -1,8 +1,8 @@
 MAKEFLAGS += --no-print-directory
 
-COMPOSE_RUN = USER_ID=$$(id -u) GROUP_ID=$$(id -g) docker compose run glue
+COMPOSE_RUN = USER_ID=$$(id -u) docker compose run glue
 
-COMPOSE_EXEC = USER_ID=$$(id -u) GROUP_ID=$$(id -g) docker compose exec glue
+COMPOSE_EXEC = USER_ID=$$(id -u) docker compose exec glue
 
 
 .PHONY: help
@@ -132,7 +132,7 @@ endif
 .PHONY: clean-notebooks
 clean-notebooks: ## Removes output cells from Jupyter notebooks
 ifeq ($(PLATFORM), docker)
-	@jupyter nbconvert --clear-output notebooks/**.ipynb
+	@jupyter nbconvert --clear-output notebooks/**/*.ipynb
 else
 	@$(COMPOSE_RUN) -c "make clean-notebooks"
 endif

docker-compose.yml

@@ -9,7 +9,6 @@ services:
       dockerfile: Dockerfile
       args:
         - USER_ID=${USER_ID:-}
-        - GROUP_ID=${GROUP_ID:-}
     volumes:
       - ~/.aws:/home/glue_user/.aws
       - .:/home/glue_user/workspace

notebooks/sample/notebook.ipynb

@@ -0,0 +1,52 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e77e0d17",
+   "metadata": {},
+   "source": [
+    "This script is taken from the [AWS Glue documentation](https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-libraries.html#develop-local-docker-image)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "16d841a2-02e6-4f4a-9f4d-29d03f0cbf79",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from src.sample.script import GluePythonSampleTest\n",
+    "\n",
+    "\n",
+    "GluePythonSampleTest().run()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4246de43-b703-4e36-8cc9-6fd2c22a8076",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "PySpark",
+   "language": "python",
+   "name": "pysparkkernel"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "python",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "pyspark",
+   "pygments_lexer": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -31,7 +31,7 @@ pyspark = "3.3.0"
 line-length = 88
 output-format = "full"
 respect-gitignore = true
-target-version = "py37"
+target-version = "py310"
 
 [tool.ruff.lint]
 select = ["ALL"]