test: Restore coverage measurement

.dockerignore

@@ -1,9 +1,11 @@
 __pycache__
+.coverage*
 .env
-.env.sample
 .ipynb_checkpoints
 .mypy_cache
 .pytest_cache
 .ruff_cache
 .venv
+coverage
+htmlcov
 mypy-results

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
         uses: addnab/docker-run-action@v3
         with:
           image: amazon/aws-glue-libs:glue_libs_4.0.0_image_01
-          options: --volume ${{ github.workspace }}:/home/glue_user/workspace
+          options: --volume ${{ github.workspace }}:/home/glue_user/workspace --env AWS_REGION=us-east-1
           run: |
             pip3 install --no-warn-script-location --user --upgrade pip==24.0 
             pip3 install --no-warn-script-location --user -r requirements.txt

.gitignore

@@ -1,8 +1,11 @@
 __pycache__
+.coverage*
 .env
 .ipynb_checkpoints
 .mypy_cache
 .pytest_cache
 .ruff_cache
 .venv
-mypy-results
+coverage
+htmlcov
+mypy-results

.vscode/settings.json

@@ -19,5 +19,6 @@
   "python.analysis.diagnosticSeverityOverrides": {
     "reportGeneralTypeIssues": "warning"
   },
-  "python.testing.pytestEnabled": true
+  "python.testing.pytestEnabled": true,
+  "autoDocstring.docstringFormat": "numpy-notypes",
 }

Makefile

@@ -99,6 +99,16 @@ else
 endif
 
 
+.PHONY: coverage
+coverage: ## Generate test coverage HTML report
+ifeq ($(PLATFORM), docker)
+	@pytest --cov=src --cov=glue_utils --cov-branch --cov-report=term
+	@coverage html
+else
+	@$(COMPOSE_RUN) -c "make coverage"
+endif
+
+
 .PHONY: checks
 checks: format typecheck
 

README.md

@@ -95,6 +95,7 @@ format               Format project source code
 lint                 Check source code for common errors
 typecheck            Check type annotations
 test                 Run automated tests
+coverage             Generate test coverage HTML report
 shell                Start a bash shell session inside the container
 clean-notebooks      Removes output cells from Jupyter notebooks
 pyspark              Start a Spark shell session

glue_utils/context.py

@@ -4,26 +4,50 @@
 
 from awsglue.context import GlueContext
 from awsglue.job import Job
-from pyspark import SparkConf
-from pyspark.sql import SparkSession
+from pyspark import SparkConf, SparkContext
 
 
 class ManagedGlueContext(ContextDecorator):
-    options: dict[str, str]
+    job_options: dict[str, str]
+    spark_conf: SparkConf | None
     job: Job
 
-    def __init__(self, options: dict[str, str] | None = None) -> None:
-        self.options = options or {}
+    def __init__(
+        self,
+        *,
+        job_options: dict[str, str] | None = None,
+        spark_conf: SparkConf | None = None,
+    ) -> None:
+        """Creates a context manager that wraps a GlueContext and
+        ensures that Job.commit() is called.
+
+        Parameters
+        ----------
+        init_options, optional
+            Dictionary of key-value pairs to pass to Job.init().
+            Defaults to None.
+        conf, optional
+            Custom SparkConf to use with SparkContext.getOrCreate().
+            Defaults to None.
+        """
+        self.job_options = job_options or {}
+        self.spark_conf = spark_conf
 
         super().__init__()
 
     def __enter__(self) -> GlueContext:
-        glue_context = self.create_glue_context(self.create_spark_session())
+        job_name = self.job_options.get("JOB_NAME", "")
+
+        conf = self.spark_conf or SparkConf()
+        conf = conf.setAppName(job_name)
 
-        self.job = Job(glue_context)
-        self.job.init(self.options.get("JOB_NAME", ""), self.options)
+        spark_context = SparkContext.getOrCreate(conf)
+        self.glue_context = GlueContext(spark_context)
 
-        return glue_context
+        self.job = Job(self.glue_context)
+        self.job.init(job_name, self.job_options)
+
+        return self.glue_context
 
     def __exit__(
         self,
@@ -34,18 +58,3 @@ def __exit__(
         self.job.commit()
 
         return cast(bool, False)  # noqa: FBT003
-
-    def create_spark_conf(self) -> SparkConf:
-        return SparkConf()
-
-    def create_spark_session(self) -> SparkSession:
-        conf = self.create_spark_conf()
-
-        return (
-            SparkSession.builder.appName(name=self.options.get("JOB_NAME", ""))
-            .config(conf=conf)
-            .getOrCreate()
-        )
-
-    def create_glue_context(self, spark_session: SparkSession) -> GlueContext:
-        return GlueContext(spark_session)

pyproject.toml

@@ -18,6 +18,7 @@ pip = "^24.0"
 pytest-randomly = "^3.15.0"
 pytest = "^8.0.1"
 ruff = "^0.2.2"
+pytest-cov = "^4.1.0"
 
 [tool.poetry.group.runtime.dependencies]
 # The "runtime" dependency group refers to dependencies that already 
@@ -80,4 +81,25 @@ disallow_untyped_defs = false
 disallow_untyped_calls = false
 
 [tool.pytest.ini_options]
-filterwarnings = ["ignore::DeprecationWarning"]
+filterwarnings = [
+    "ignore::DeprecationWarning", 
+    "ignore::FutureWarning:.*pyspark.*",
+]
+
+[tool.coverage.run]
+omit = ["**/test_*.py"]
+
+[tool.coverage.report]
+skip_empty = true
+exclude_also = [
+    "def __repr__",
+    "if self.debug:",
+    "if settings.DEBUG",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if 0:",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]