diff --git a/alembic/env.py b/alembic/env.py index 7a552553..1748e160 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,3 +1,4 @@ +import logging import os from logging.config import fileConfig @@ -10,27 +11,30 @@ config = context.config # Support RSTUF Worker Container multiple environment variables for SQL +# TODO: Removed deprecated RSTUF_SQL_* environment variables in the future sql_server = os.getenv("RSTUF_SQL_SERVER") sql_user = os.getenv("RSTUF_SQL_USER") sql_password = os.getenv("RSTUF_SQL_PASSWORD") - -if sql_server is None: - raise ValueError("RSTUF_SQL_SERVER is required") - -sql_server = sql_server.replace("postgresql://", "") # remove protocol - -if sql_user and sql_password is None: - raise ValueError( - "RSTUF_SQL_PASSWORD is required when using RSTUF_SQL_USER" - ) -elif sql_user and sql_password: - if sql_password.startswith("/run/secrets"): - with open(sql_password) as f: - sql_password = f.read().rstrip("\n") - sql_server_uri = f"postgresql://{sql_user}:{sql_password}@{sql_server}" +if sql_server or sql_user or sql_password: + logging.warning("RSTUF_SQL_* are deprecated. Use RSTUF_DB_* instead.") +db_server = os.getenv("RSTUF_DB_SERVER", sql_server) +db_user = os.getenv("RSTUF_DB_USER", sql_user) +db_password = os.getenv("RSTUF_DB_PASSWORD", sql_password) + +if db_server is None: + raise ValueError("RSTUF_DB_SERVER is required") + +db_server = db_server.replace("postgresql://", "") # remove protocol + +if db_user and db_password is None: + raise ValueError("RSTUF_DB_PASSWORD is required when using RSTUF_DB_USER") +elif db_user and db_password: + if db_password.startswith("/run/secrets"): + with open(db_password) as f: + db_password = f.read().rstrip("\n") + sql_server_uri = f"postgresql://{db_user}:{db_password}@{db_server}" else: - sql_server_uri = f"postgresql://{sql_server}" - + sql_server_uri = f"postgresql://{db_server}" config.set_main_option("sqlalchemy.url", sql_server_uri) diff --git a/docker-compose-aws.yml b/docker-compose-aws.yml index 569c7ac6..f8a35da2 100644 --- a/docker-compose-aws.yml +++ b/docker-compose-aws.yml @@ -70,7 +70,7 @@ services: - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage - RSTUF_BROKER_SERVER=redis://redis - RSTUF_REDIS_SERVER=redis://redis - - RSTUF_SQL_SERVER=postgres:secret@postgres:5432 + - RSTUF_DB_SERVER=postgres:secret@postgres:5432 - METADATA_BASE_URL="http://localstack:4566/tuf-metadata/" volumes: - ./:/opt/repository-service-tuf-worker:z diff --git a/docker-compose-redis.yml b/docker-compose-redis.yml index 4f30d50b..7aab6803 100644 --- a/docker-compose-redis.yml +++ b/docker-compose-redis.yml @@ -62,7 +62,7 @@ services: - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage - RSTUF_BROKER_SERVER=redis://redis - RSTUF_REDIS_SERVER=redis://redis - - RSTUF_SQL_SERVER=postgres:secret@postgres:5432 + - RSTUF_DB_SERVER=postgres:secret@postgres:5432 volumes: - ./:/opt/repository-service-tuf-worker:z - repository-service-tuf-storage:/var/opt/repository-service-tuf/storage diff --git a/docker-compose.yml b/docker-compose.yml index f7caf038..94bc2da0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,9 +72,9 @@ services: - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage - RSTUF_BROKER_SERVER=amqp://guest:guest@rabbitmq:5672 - - RSTUF_SQL_SERVER=postgres:5432 - - RSTUF_SQL_USER=postgres - - RSTUF_SQL_PASSWORD=secret + - RSTUF_DB_SERVER=postgres:5432 + - RSTUF_DB_USER=postgres + - RSTUF_DB_PASSWORD=secret - RSTUF_REDIS_SERVER=redis://redis volumes: - ./:/opt/repository-service-tuf-worker:z diff --git a/docs/source/guide/Docker_README.md b/docs/source/guide/Docker_README.md index 8e054df7..f56fec27 100644 --- a/docs/source/guide/Docker_README.md +++ b/docs/source/guide/Docker_README.md @@ -43,7 +43,7 @@ docker run --env="RSTUF_STORAGE_BACKEND=LocalStorage" \ --env="RSTUF_LOCAL_STORAGE_BACKEND_PATH=/metadata" \ --env="RSTUF_BROKER_SERVER=guest:guest@rabbitmq:5672" \ --env="RSTUF_REDIS_SERVER=redis://redis" \ - --env="RSTUF_SQL_SERVER=postgresql://postgres:secret@postgres:5432" \ + --env="RSTUF_DB_SERVER=postgresql://postgres:secret@postgres:5432" \ ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest \ ``` @@ -68,7 +68,7 @@ The result backend must to be compatible with Celery. See Example: `redis://redis` -#### (Required) `RSTUF_SQL_SERVER` +#### (Deprecated see `RSTUF_DB_SERVER`) `RSTUF_SQL_SERVER` RSTUF requires [PostgreSQL](https://www.postgresql.org). @@ -76,13 +76,14 @@ Example: `postgres:secret@postgres:5432` * Optional variables: - * `RSTUF_SQL_USER` optional information about the user name + * `RSTUF_SQL_USER` (Deprecated see `RSTUF_DB_USER`) optional information + about the user name If using this optional variable: - Do not include the user in the `RSTUF_SQL_SERVER`. - The `RSTUF_SQL_PASSWORD` becomes required - * `RSTUF_SQL_PASSWORD` use this variable to provide the password separately. + * `RSTUF_SQL_PASSWORD` (Deprecated see `RSTUF_DB_PASSWORD`) use this variable to provide the password separately. - Do not include the password in the `RSTUF_SQL_SERVER` - This environment variable supports container secrets when the `/run/secrets` volume is added to the path. @@ -94,6 +95,32 @@ Example: `postgres:secret@postgres:5432` RSTUF_SQL_PASSWORD=/run/secrets/POSTGRES_PASSWORD ``` +#### (Required) `RSTUF_DB_SERVER` + +RSTUF requires [PostgreSQL](https://www.postgresql.org). + +Example: `postgres:secret@postgres:5432` + +* Optional variables: + + * `RSTUF_DB_USER` optional information about the user name + + If using this optional variable: + - Do not include the user in the `RSTUF_DB_SERVER`. + - The `RSTUF_DB_PASSWORD` becomes required + + * `RSTUF_DB_PASSWORD` use this variable to provide the password separately. + - Do not include the password in the `RSTUF_DB_SERVER` + - This environment variable supports container secrets when the `/run/secrets` + volume is added to the path. + + Example: + ``` + RSTUF_DB_SERVER=sqlserver:5432 + RSTUF_DB_USER=postgres + RSTUF_DB_PASSWORD=/run/secrets/POSTGRES_PASSWORD + ``` + #### (Optional) `RSTUF_REDIS_SERVER_PORT` Redis Server port number. Default: 6379 diff --git a/repository_service_tuf_worker/models/__init__.py b/repository_service_tuf_worker/models/__init__.py index 45690a5f..af6e5792 100644 --- a/repository_service_tuf_worker/models/__init__.py +++ b/repository_service_tuf_worker/models/__init__.py @@ -16,8 +16,8 @@ ) -def rstuf_db(sql_server: str) -> sessionmaker: - engine = create_engine(sql_server) +def rstuf_db(db_server: str) -> sessionmaker: + engine = create_engine(db_server) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) # Base.metadata.create_all(bind=engine) diff --git a/repository_service_tuf_worker/repository.py b/repository_service_tuf_worker/repository.py index e7bd5761..cb057fe0 100644 --- a/repository_service_tuf_worker/repository.py +++ b/repository_service_tuf_worker/repository.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Repository Service for TUF Contributors +# SPDX-FileCopyrightText: 2023-2024 Repository Service for TUF Contributors # SPDX-FileCopyrightText: 2022-2023 VMware Inc # # SPDX-License-Identifier: MIT @@ -162,27 +162,43 @@ def refresh_settings(self, worker_settings: Optional[Dynaconf] = None): else: settings = worker_settings # - # SQL + # DB # + # TODO: Removed deprecated RSTUF_SQL_* settings in the future sql_server_url = self._worker_settings.get("SQL_SERVER") + sql_user = self._worker_settings.get("SQL_USER") + sql_password = self._worker_settings.get("SQL_PASSWORD") + if sql_server_url or sql_user or sql_password: + logging.warning( + "Using RSTUF_SQL_* environment variables is deprecated. " + "Use RSTUF_DB_* instead." + ) + db_server_url = self._worker_settings.get("DB_SERVER", sql_server_url) + db_user = self._worker_settings.get("DB_USER", sql_user) + db_password = self._worker_settings.get("DB_PASSWORD", sql_password) + # clean 'postgresql://' if present - sql_server = sql_server_url.replace("postgresql://", "") - if sql_user := self._worker_settings.get("SQL_USER"): - if self._worker_settings.SQL_PASSWORD.startswith("/run/secrets"): + db_server = db_server_url.replace("postgresql://", "") + if db_user := self._worker_settings.get("DB_USER", sql_user): + if db_password is None: + raise AttributeError( + "'Settings' object has no attribute 'DB_PASSWORD'" + "'DB_PASSWORD' is required when using 'DB_USER'" + ) + if db_password.startswith("/run/secrets"): try: - with open(self._worker_settings.SQL_PASSWORD) as f: - sql_password = f.read().rstrip("\n") + with open(db_password) as f: + db_password = f.read().rstrip("\n") except OSError as err: logging.error(str(err)) raise err - else: - sql_password = self._worker_settings.SQL_PASSWORD - settings.SQL = rstuf_db( - f"postgresql://{sql_user}:{sql_password}@{sql_server}" - ) + + db_uri = f"postgresql://{db_user}:{db_password}@{db_server}" else: - settings.SQL = rstuf_db(f"postgresql://{sql_server}") + db_uri = f"postgresql://{db_server}" + logging.info(f"DB URI: {db_uri}") + settings.SQL = rstuf_db(db_uri) # # Backends # diff --git a/tests/unit/tuf_repository_service_worker/test_repository.py b/tests/unit/tuf_repository_service_worker/test_repository.py index 62233269..36389c21 100644 --- a/tests/unit/tuf_repository_service_worker/test_repository.py +++ b/tests/unit/tuf_repository_service_worker/test_repository.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023 Repository Service for TUF Contributors +# SPDX-FileCopyrightText: 2023-2024 Repository Service for TUF Contributors # SPDX-FileCopyrightText: 2022-2023 VMware Inc # # SPDX-License-Identifier: MIT @@ -181,9 +181,9 @@ def test_write_repository_settings(self, monkeypatch, test_repo): ] def test_refresh_settings_with_sql_user_password(self, test_repo): - test_repo._worker_settings.SQL_SERVER = "fake-sql:5433" - test_repo._worker_settings.SQL_USER = "psql" - test_repo._worker_settings.SQL_PASSWORD = "psqlpass" + test_repo._worker_settings.DB_SERVER = "fake-sql:5433" + test_repo._worker_settings.DB_USER = "psql" + test_repo._worker_settings.DB_PASSWORD = "psqlpass" fake_sql = pretend.stub() repository.rstuf_db = pretend.call_recorder(lambda *a: fake_sql) @@ -195,20 +195,20 @@ def test_refresh_settings_with_sql_user_password(self, test_repo): ] def test_refresh_settings_with_sql_user_missing_password(self, test_repo): - test_repo._worker_settings.SQL_SERVER = "fake-sql:5433" - test_repo._worker_settings.SQL_USER = "psql" + test_repo._worker_settings.DB_SERVER = "fake-sql:5433" + test_repo._worker_settings.DB_USER = "psql" with pytest.raises(AttributeError) as e: test_repo.refresh_settings() - assert "'Settings' object has no attribute 'SQL_PASSWORD'" in str(e) + assert "'Settings' object has no attribute 'DB_PASSWORD'" in str(e) def test_refresh_settings_with_sql_user_password_secrets( self, test_repo, monkeypatch ): - test_repo._worker_settings.SQL_SERVER = "fake-sql:5433" - test_repo._worker_settings.SQL_USER = "psql" - test_repo._worker_settings.SQL_PASSWORD = "/run/secrets/SQL_PASSWORD" + test_repo._worker_settings.DB_SERVER = "fake-sql:5433" + test_repo._worker_settings.DB_USER = "psql" + test_repo._worker_settings.DB_PASSWORD = "/run/secrets/DB_PASSWORD" fake_data = pretend.stub( read=pretend.call_recorder(lambda: "psqlpass\n") ) @@ -234,9 +234,9 @@ def test_refresh_settings_with_sql_user_password_secrets_OSError( self, test_repo, monkeypatch, caplog ): caplog.set_level(repository.logging.ERROR) - test_repo._worker_settings.SQL_SERVER = "fake-sql:5433" - test_repo._worker_settings.SQL_USER = "psql" - test_repo._worker_settings.SQL_PASSWORD = "/run/secrets/SQL_PASSWORD" + test_repo._worker_settings.DB_SERVER = "fake-sql:5433" + test_repo._worker_settings.DB_USER = "psql" + test_repo._worker_settings.DB_PASSWORD = "/run/secrets/DB_PASSWORD" monkeypatch.setitem( repository.__builtins__, "open", @@ -249,6 +249,18 @@ def test_refresh_settings_with_sql_user_password_secrets_OSError( assert "No permission /run/secrets/*" in str(e) assert "No permission /run/secrets/*" == caplog.messages[0] + def test_refresh_settings_with_deprecated_sql( + self, test_repo, monkeypatch, caplog + ): + caplog.set_level(repository.logging.WARNING) + test_repo._worker_settings.SQL_SERVER = "fake-sql:5433" + + test_repo.refresh_settings() + assert ( + "Using RSTUF_SQL_* environment variables is deprecated. " + "Use RSTUF_DB_* instead." + ) in caplog.messages + def test__sign(self, test_repo, monkeypatch): fake_key_dict = {"keyval": "foo", "keyid": "keyid"} fake_settings = pretend.stub( diff --git a/tox.ini b/tox.ini index 1a88e309..bd78b9f3 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ setenv = RSTUF_WORKER_ID = "test" RSTUF_BROKER_SERVER = fakeserver RSTUF_REDIS_SERVER = redis://fake-redis - RSTUF_SQL_SERVER = postgresql://fake-sql + RSTUF_DB_SERVER = postgresql://fake-sql RSTUF_STORAGE_BACKEND = LocalStorage RSTUF_LOCAL_STORAGE_BACKEND_PATH = ./data-test/s DATA_DIR = ./data-test