Skip to content

Commit

Permalink
move functions in settings to dedicated submodule for better testability
Browse files Browse the repository at this point in the history
  • Loading branch information
mathiasertl committed Oct 8, 2023
1 parent c54967d commit 4bdc7e6
Show file tree
Hide file tree
Showing 21 changed files with 630 additions and 298 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ jobs:
django-version: [ "3.2", "4.2" ]
cryptography-version: [ "41.0" ]

env:
DJANGO_CA_SECRET_KEY: dummy

name: Python ${{ matrix.python-version }}, Django ${{ matrix.django-version }}, cryptography ${{ matrix.cryptography-version }}
steps:

Expand Down
122 changes: 20 additions & 102 deletions ca/ca/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@

import os
from pathlib import Path
from typing import List, Optional, Tuple
from typing import List

from django.core.exceptions import ImproperlyConfigured

try:
import yaml
except ImportError:
yaml = False # type: ignore[assignment]
from ca.settings_utils import (
load_secret_key,
load_settings_from_environment,
load_settings_from_files,
update_database_setting_from_environment,
)

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = Path(__file__).resolve().parent.parent # ca/
Expand Down Expand Up @@ -206,66 +206,22 @@
},
}

# Load settings from files
for _setting, _value in load_settings_from_files(BASE_DIR):
globals()[_setting] = _value

# CONFIGURATION_DIRECTORY is set by the SystemD ConfigurationDirectory= directive.
SETTINGS_DIRS = os.environ.get("DJANGO_CA_SETTINGS", os.environ.get("CONFIGURATION_DIRECTORY", ""))


def _get_settings_files(base_dir: Path, dirs: str) -> List[Tuple[str, Path]]:
settings_files: List[Tuple[str, Path]] = []

for path in [base_dir / p for p in dirs.split(":")]:
if not path.exists():
raise ImproperlyConfigured(f"{path}: No such file or directory.")

if path.is_dir():
# exclude files that don't end with '.yaml' and any directories
settings_files += sorted(
[(_f.name, path) for _f in path.iterdir() if _f.suffix == ".yaml" and not _f.is_dir()]
)
else:
settings_files.append((path.name, path.parent))

settings_yaml = base_dir / "ca" / "settings.yaml"
if settings_yaml.exists():
settings_files.append((settings_yaml.name, settings_yaml.parent))

return settings_files


if not _skip_local_config and yaml is not False: # type: ignore[comparison-overlap]
_settings_files = _get_settings_files(BASE_DIR, SETTINGS_DIRS)
# Load settings from environment variables
for _setting, _value in load_settings_from_environment():
globals()[_setting] = _value

for _filename, _filename_path in _settings_files:
_full_path = _filename_path / _filename
with open(_full_path, encoding="utf-8") as stream:
data = yaml.safe_load(stream)
if data is None:
pass # silently ignore empty files
elif not isinstance(data, dict):
raise ImproperlyConfigured(f"{_full_path}: File is not a key/value mapping.")
else:
for key, value in data.items():
globals()[key] = value


def _parse_bool(env_value: str) -> bool:
# parse an env variable that is supposed to represent a boolean value
return env_value.strip().lower() in ("true", "yes", "1")


# Also use DJANGO_CA_ environment variables
for key, value in {k[10:]: v for k, v in os.environ.items() if k.startswith("DJANGO_CA_")}.items():
if key == "SETTINGS": # points to yaml files loaded above
continue

if key == "ALLOWED_HOSTS":
globals()[key] = value.split()
elif key in ("CA_USE_CELERY", "CA_ENABLE_ACME", "CA_ENABLE_REST_API", "ENABLE_ADMIN"):
globals()[key] = _parse_bool(value)
else:
globals()[key] = value
# Try to use POSTGRES_* and MYSQL_* environment variables to determine database access credentials.
# These are the variables set by the standard PostgreSQL/MySQL Docker containers.
update_database_setting_from_environment(DATABASES)

# Load SECRET_KEY from a file if not already defined.
# NOTE: This must be called AFTER load_settings_from_environment(), as this might set SECRET_KEY_FILE in the
# first place.
SECRET_KEY = load_secret_key(SECRET_KEY, SECRET_KEY_FILE)

if CA_ENABLE_CLICKJACKING_PROTECTION is True:
if "django.middleware.clickjacking.XFrameOptionsMiddleware" not in MIDDLEWARE:
Expand All @@ -275,15 +231,6 @@ def _parse_bool(env_value: str) -> bool:
if not ALLOWED_HOSTS and CA_DEFAULT_HOSTNAME:
ALLOWED_HOSTS = [CA_DEFAULT_HOSTNAME]

if not SECRET_KEY:
# We generate SECRET_KEY on first invocation
if not SECRET_KEY_FILE:
SECRET_KEY_FILE = os.environ.get("DJANGO_CA_SECRET_KEY_FILE", "/var/lib/django-ca/secret_key")

if SECRET_KEY_FILE and os.path.exists(SECRET_KEY_FILE):
with open(SECRET_KEY_FILE, encoding="utf-8") as stream:
SECRET_KEY = stream.read()

# Remove django.contrib.admin if the admin interface is not enabled.
if ENABLE_ADMIN is not True and "django.contrib.admin" in INSTALLED_APPS:
INSTALLED_APPS.remove("django.contrib.admin")
Expand All @@ -292,35 +239,6 @@ def _parse_bool(env_value: str) -> bool:
if CA_ENABLE_REST_API and "ninja" not in INSTALLED_APPS:
INSTALLED_APPS.append("ninja")


def _set_db_setting(name: str, env_name: str, default: Optional[str] = None) -> None:
if DATABASES["default"].get(name):
return

if os.environ.get(env_name):
DATABASES["default"][name] = os.environ[env_name]
elif os.environ.get(f"{env_name}_FILE"):
with open(os.environ[f"{env_name}_FILE"], encoding="utf-8") as env_stream:
DATABASES["default"][name] = env_stream.read()
elif default is not None:
DATABASES["default"][name] = default


# use POSTGRES_* environment variables from the postgres Docker image
if DATABASES["default"]["ENGINE"] in (
"django.db.backends.postgresql_psycopg2",
"django.db.backends.postgresql",
):
_set_db_setting("PASSWORD", "POSTGRES_PASSWORD", default="postgres")
_set_db_setting("USER", "POSTGRES_USER", default="postgres")
_set_db_setting("NAME", "POSTGRES_DB", default=DATABASES["default"].get("USER"))

# use MYSQL_* environment variables from the mysql Docker image
if DATABASES["default"]["ENGINE"] == "django.db.backends.mysql":
_set_db_setting("PASSWORD", "MYSQL_PASSWORD")
_set_db_setting("USER", "MYSQL_USER")
_set_db_setting("NAME", "MYSQL_DATABASE")

if LOGGING is None:
LOGGING = {
"version": 1,
Expand Down
139 changes: 139 additions & 0 deletions ca/ca/settings_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
# This file is part of django-ca (https://github.com/mathiasertl/django-ca).
#
# django-ca is free software: you can redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# django-ca is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License along with django-ca. If not, see
# <http://www.gnu.org/licenses/>.

"""Utility functions for loading settings."""
import logging
import os
from pathlib import Path
from typing import Any, Dict, Iterator, Optional, Tuple

from django.core.exceptions import ImproperlyConfigured

try:
import yaml
except ImportError: # pragma: no cover
yaml = False # type: ignore[assignment]


def load_secret_key(secret_key: Optional[str], secret_key_file: Optional[str]) -> str:
"""Load SECRET_KEY from file if not set elsewhere."""
if secret_key:
return secret_key

if secret_key_file and os.path.exists(secret_key_file):
with open(secret_key_file, encoding="utf-8") as stream:
return stream.read()
raise ImproperlyConfigured("Unable to determine SECRET_KEY.")


def get_settings_files(base_dir: Path, paths: str) -> Iterator[Path]:
"""Get relevant settings files."""
for path in [base_dir / p for p in paths.split(":")]:
if not path.exists():
raise ImproperlyConfigured(f"{path}: No such file or directory.")

if path.is_dir():
# exclude files that don't end with '.yaml' and any directories
yield from sorted(
[path / _f.name for _f in path.iterdir() if _f.suffix == ".yaml" and not _f.is_dir()]
)
else:
yield path

settings_yaml = base_dir / "ca" / "settings.yaml"
if settings_yaml.exists():
yield settings_yaml


def load_settings_from_files(base_dir: Path) -> Iterator[Tuple[str, Any]]:
"""Load settings from YAML files."""
# TYPEHINT NOTE: mypy typehints this to a module in the initial import statement
if yaml is False: # type: ignore[comparison-overlap]
return

# CONFIGURATION_DIRECTORY is set by the SystemD ConfigurationDirectory= directive.
settings_paths = os.environ.get("DJANGO_CA_SETTINGS", os.environ.get("CONFIGURATION_DIRECTORY", ""))

settings_files = []

for full_path in get_settings_files(base_dir, settings_paths):
with open(full_path, encoding="utf-8") as stream:
try:
data = yaml.safe_load(stream)
except Exception as ex:
logging.exception(ex)
raise ImproperlyConfigured(f"{full_path}: Invalid YAML.") from ex

if data is None:
pass # silently ignore empty files
elif not isinstance(data, dict):
raise ImproperlyConfigured(f"{full_path}: File is not a key/value mapping.")
else:
settings_files.append(full_path)
for key, value in data.items():
yield key, value

# ALSO yield the SETTINGS_FILES setting with the loaded files.
yield "SETTINGS_FILES", tuple(settings_files)


def load_settings_from_environment() -> Iterator[Tuple[str, Any]]:
"""Load settings from the environment."""
for key, value in {k[10:]: v for k, v in os.environ.items() if k.startswith("DJANGO_CA_")}.items():
if key == "SETTINGS": # points to yaml files loaded in get_settings_files
continue

if key == "ALLOWED_HOSTS":
yield key, value.split()
elif key in ("CA_USE_CELERY", "CA_ENABLE_ACME", "CA_ENABLE_REST_API", "ENABLE_ADMIN"):
yield key, parse_bool(value)
else:
yield key, value


def parse_bool(value: str) -> bool:
"""Parse a variable that is supposed to represent a boolean value."""
return value.strip().lower() in ("true", "yes", "1")


def _set_db_setting(
databases: Dict[str, Dict[str, Any]], name: str, env_name: str, default: Optional[str] = None
) -> None:
if databases["default"].get(name):
return

if os.environ.get(env_name):
databases["default"][name] = os.environ[env_name]
elif os.environ.get(f"{env_name}_FILE"):
with open(os.environ[f"{env_name}_FILE"], encoding="utf-8") as env_stream:
databases["default"][name] = env_stream.read()
elif default is not None:
databases["default"][name] = default


def update_database_setting_from_environment(databases: Dict[str, Dict[str, Any]]) -> None:
"""Update the DATABASES dict with Docker-style environment variables."""
# use POSTGRES_* environment variables from the postgres Docker image
if databases["default"]["ENGINE"] in (
"django.db.backends.postgresql_psycopg2",
"django.db.backends.postgresql",
):
_set_db_setting(databases, "PASSWORD", "POSTGRES_PASSWORD", default="postgres")
_set_db_setting(databases, "USER", "POSTGRES_USER", default="postgres")
_set_db_setting(databases, "NAME", "POSTGRES_DB", default=databases["default"].get("USER"))

# use MYSQL_* environment variables from the mysql Docker image
if databases["default"]["ENGINE"] == "django.db.backends.mysql":
_set_db_setting(databases, "PASSWORD", "MYSQL_PASSWORD")
_set_db_setting(databases, "USER", "MYSQL_USER")
_set_db_setting(databases, "NAME", "MYSQL_DATABASE")
53 changes: 4 additions & 49 deletions ca/ca/test_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,10 @@
"""Test settings for the django-ca project."""

import json
import os
import sys
from importlib.metadata import version
from pathlib import Path

import packaging.version

import cryptography

import django

# Base paths in this project
BASE_DIR = Path(__file__).resolve().parent.parent # ca/
ROOT_DIR = Path(BASE_DIR).parent # git repository root
DOC_DIR = ROOT_DIR / "docs" / "source"
FIXTURES_DIR = BASE_DIR / "django_ca" / "tests" / "fixtures"

DEBUG = False

Expand Down Expand Up @@ -184,8 +172,10 @@
},
}

# Fixture data used by test cases
with open(FIXTURES_DIR / "cert-data.json", encoding="utf-8") as stream:
# Load fixture data in settings as well. We cannot load it from django_ca.tests.base.constants, as that would
# import the parent modules, which at present also "from django.conf import settings", which causes a circular
# import situation.
with open(BASE_DIR / "django_ca" / "tests" / "fixtures" / "cert-data.json", encoding="utf-8") as stream:
_fixture_data = json.load(stream)

# Custom settings
Expand Down Expand Up @@ -237,41 +227,6 @@
}
CA_ENABLE_ACME = True

# Newest versions of software components.
# NOTE: These values are validated by various release scripts
NEWEST_PYTHON_VERSION = (3, 11)
NEWEST_CRYPTOGRAPHY_VERSION = (41, 0)
NEWEST_DJANGO_VERSION = (4, 2)
NEWEST_ACME_VERSION = (2, 6, 0)

# Determine if we're running on the respective newest versions
_parsed_cg_version = packaging.version.parse(cryptography.__version__).release
CRYPTOGRAPHY_VERSION = _parsed_cg_version[:2]
ACME_VERSION = packaging.version.parse(version("acme")).release

NEWEST_PYTHON = sys.version_info[0:2] == NEWEST_PYTHON_VERSION
NEWEST_CRYPTOGRAPHY = CRYPTOGRAPHY_VERSION == NEWEST_CRYPTOGRAPHY_VERSION
NEWEST_DJANGO = django.VERSION[:2] == NEWEST_DJANGO_VERSION
NEWEST_ACME = ACME_VERSION == NEWEST_ACME_VERSION
NEWEST_VERSIONS = NEWEST_PYTHON and NEWEST_CRYPTOGRAPHY and NEWEST_DJANGO and NEWEST_ACME

# Only run Selenium tests if we use the newest Python, cryptography and acme.
RUN_SELENIUM_TESTS = NEWEST_PYTHON and NEWEST_CRYPTOGRAPHY and NEWEST_ACME

# Set COLUMNS, which is used by argparse to determine the terminal width. If this is not set, the output of
# some argparse commands depend on the terminal size.
os.environ["COLUMNS"] = "80"

if "GECKOWEBDRIVER" in os.environ:
GECKODRIVER_PATH = os.path.join(os.environ["GECKOWEBDRIVER"], "geckodriver")
else:
GECKODRIVER_PATH = os.path.join(ROOT_DIR, "contrib", "selenium", "geckodriver")

if "TOX_ENV_DIR" in os.environ:
GECKODRIVER_LOG_PATH = os.path.join(os.environ["TOX_ENV_DIR"], "geckodriver.log")
else:
GECKODRIVER_LOG_PATH = os.path.join(ROOT_DIR, "geckodriver.log")


CA_USE_CELERY = False
CA_ENABLE_REST_API = True
Expand Down
Loading

0 comments on commit 4bdc7e6

Please sign in to comment.