Merge branch 'main' into dependabot/github_actions/github/codeql-acti…

…on-3.25.14

alembic/env.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from logging.config import fileConfig
 
@@ -10,27 +11,30 @@
 config = context.config
 
 # Support RSTUF Worker Container multiple environment variables for SQL
+# TODO: Removed deprecated RSTUF_SQL_* environment variables in the future
 sql_server = os.getenv("RSTUF_SQL_SERVER")
 sql_user = os.getenv("RSTUF_SQL_USER")
 sql_password = os.getenv("RSTUF_SQL_PASSWORD")
-
-if sql_server is None:
-    raise ValueError("RSTUF_SQL_SERVER is required")
-
-sql_server = sql_server.replace("postgresql://", "")  # remove protocol
-
-if sql_user and sql_password is None:
-    raise ValueError(
-        "RSTUF_SQL_PASSWORD is required when using RSTUF_SQL_USER"
-    )
-elif sql_user and sql_password:
-    if sql_password.startswith("/run/secrets"):
-        with open(sql_password) as f:
-            sql_password = f.read().rstrip("\n")
-    sql_server_uri = f"postgresql://{sql_user}:{sql_password}@{sql_server}"
+if sql_server or sql_user or sql_password:
+    logging.warning("RSTUF_SQL_* are deprecated. Use RSTUF_DB_* instead.")
+db_server = os.getenv("RSTUF_DB_SERVER", sql_server)
+db_user = os.getenv("RSTUF_DB_USER", sql_user)
+db_password = os.getenv("RSTUF_DB_PASSWORD", sql_password)
+
+if db_server is None:
+    raise ValueError("RSTUF_DB_SERVER is required")
+
+db_server = db_server.replace("postgresql://", "")  # remove protocol
+
+if db_user and db_password is None:
+    raise ValueError("RSTUF_DB_PASSWORD is required when using RSTUF_DB_USER")
+elif db_user and db_password:
+    if db_password.startswith("/run/secrets"):
+        with open(db_password) as f:
+            db_password = f.read().rstrip("\n")
+    sql_server_uri = f"postgresql://{db_user}:{db_password}@{db_server}"
 else:
-    sql_server_uri = f"postgresql://{sql_server}"
-
+    sql_server_uri = f"postgresql://{db_server}"
 
 config.set_main_option("sqlalchemy.url", sql_server_uri)
 

docker-compose-aws.yml

@@ -70,7 +70,7 @@ services:
       - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage
       - RSTUF_BROKER_SERVER=redis://redis
       - RSTUF_REDIS_SERVER=redis://redis
-      - RSTUF_SQL_SERVER=postgres:secret@postgres:5432
+      - RSTUF_DB_SERVER=postgres:secret@postgres:5432
       - METADATA_BASE_URL="http://localstack:4566/tuf-metadata/"
     volumes:
       - ./:/opt/repository-service-tuf-worker:z

docker-compose-redis.yml

@@ -62,7 +62,7 @@ services:
       - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage
       - RSTUF_BROKER_SERVER=redis://redis
       - RSTUF_REDIS_SERVER=redis://redis
-      - RSTUF_SQL_SERVER=postgres:secret@postgres:5432
+      - RSTUF_DB_SERVER=postgres:secret@postgres:5432
     volumes:
       - ./:/opt/repository-service-tuf-worker:z
       - repository-service-tuf-storage:/var/opt/repository-service-tuf/storage

docker-compose.yml

@@ -72,9 +72,9 @@ services:
       - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage
       - RSTUF_ONLINE_KEY_DIR=/var/opt/repository-service-tuf/key_storage
       - RSTUF_BROKER_SERVER=amqp://guest:guest@rabbitmq:5672
-      - RSTUF_SQL_SERVER=postgres:5432
-      - RSTUF_SQL_USER=postgres
-      - RSTUF_SQL_PASSWORD=secret
+      - RSTUF_DB_SERVER=postgres:5432
+      - RSTUF_DB_USER=postgres
+      - RSTUF_DB_PASSWORD=secret
       - RSTUF_REDIS_SERVER=redis://redis
     volumes:
       - ./:/opt/repository-service-tuf-worker:z

docs/source/guide/Docker_README.md

@@ -43,7 +43,7 @@ docker run --env="RSTUF_STORAGE_BACKEND=LocalStorage" \
     --env="RSTUF_LOCAL_STORAGE_BACKEND_PATH=/metadata" \
     --env="RSTUF_BROKER_SERVER=guest:guest@rabbitmq:5672" \
     --env="RSTUF_REDIS_SERVER=redis://redis" \
-    --env="RSTUF_SQL_SERVER=postgresql://postgres:secret@postgres:5432" \
+    --env="RSTUF_DB_SERVER=postgresql://postgres:secret@postgres:5432" \
     ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest \
 ```
 
@@ -68,21 +68,22 @@ The result backend must to be compatible with Celery. See
 
 Example: `redis://redis`
 
-#### (Required) `RSTUF_SQL_SERVER`
+#### (Deprecated see `RSTUF_DB_SERVER`) `RSTUF_SQL_SERVER`
 
 RSTUF requires [PostgreSQL](https://www.postgresql.org).
 
 Example: `postgres:secret@postgres:5432`
 
 * Optional variables:
 
-  * `RSTUF_SQL_USER` optional information about the user name
+  * `RSTUF_SQL_USER` (Deprecated see `RSTUF_DB_USER`) optional information
+    about the user name
 
     If using this optional variable:
     - Do not include the user in the `RSTUF_SQL_SERVER`.
     - The `RSTUF_SQL_PASSWORD` becomes required
 
-  * `RSTUF_SQL_PASSWORD` use this variable to provide the password separately.
+  * `RSTUF_SQL_PASSWORD` (Deprecated see `RSTUF_DB_PASSWORD`) use this variable to provide the password separately.
     - Do not include the password in the `RSTUF_SQL_SERVER`
     - This environment variable supports container secrets when the `/run/secrets`
       volume is added to the path.
@@ -94,6 +95,32 @@ Example: `postgres:secret@postgres:5432`
   RSTUF_SQL_PASSWORD=/run/secrets/POSTGRES_PASSWORD
   ```
 
+#### (Required) `RSTUF_DB_SERVER`
+
+RSTUF requires [PostgreSQL](https://www.postgresql.org).
+
+Example: `postgres:secret@postgres:5432`
+
+* Optional variables:
+
+  * `RSTUF_DB_USER` optional information about the user name
+
+    If using this optional variable:
+    - Do not include the user in the `RSTUF_DB_SERVER`.
+    - The `RSTUF_DB_PASSWORD` becomes required
+
+  * `RSTUF_DB_PASSWORD` use this variable to provide the password separately.
+    - Do not include the password in the `RSTUF_DB_SERVER`
+    - This environment variable supports container secrets when the `/run/secrets`
+      volume is added to the path.
+
+  Example:
+  ```
+  RSTUF_DB_SERVER=sqlserver:5432
+  RSTUF_DB_USER=postgres
+  RSTUF_DB_PASSWORD=/run/secrets/POSTGRES_PASSWORD
+  ```
+
 #### (Optional) `RSTUF_REDIS_SERVER_PORT`
 
 Redis Server port number. Default: 6379

repository_service_tuf_worker/models/__init__.py

@@ -16,8 +16,8 @@
 )
 
 
-def rstuf_db(sql_server: str) -> sessionmaker:
-    engine = create_engine(sql_server)
+def rstuf_db(db_server: str) -> sessionmaker:
+    engine = create_engine(db_server)
     SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
     # Base.metadata.create_all(bind=engine)

repository_service_tuf_worker/repository.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023 Repository Service for TUF Contributors
+# SPDX-FileCopyrightText: 2023-2024 Repository Service for TUF Contributors
 # SPDX-FileCopyrightText: 2022-2023 VMware Inc
 #
 # SPDX-License-Identifier: MIT
@@ -162,27 +162,43 @@ def refresh_settings(self, worker_settings: Optional[Dynaconf] = None):
         else:
             settings = worker_settings
         #
-        # SQL
+        # DB
         #
+        # TODO: Removed deprecated RSTUF_SQL_* settings in the future
         sql_server_url = self._worker_settings.get("SQL_SERVER")
+        sql_user = self._worker_settings.get("SQL_USER")
+        sql_password = self._worker_settings.get("SQL_PASSWORD")
+        if sql_server_url or sql_user or sql_password:
+            logging.warning(
+                "Using RSTUF_SQL_* environment variables is deprecated. "
+                "Use RSTUF_DB_* instead."
+            )
+        db_server_url = self._worker_settings.get("DB_SERVER", sql_server_url)
+        db_user = self._worker_settings.get("DB_USER", sql_user)
+        db_password = self._worker_settings.get("DB_PASSWORD", sql_password)
+
         # clean 'postgresql://' if present
-        sql_server = sql_server_url.replace("postgresql://", "")
-        if sql_user := self._worker_settings.get("SQL_USER"):
-            if self._worker_settings.SQL_PASSWORD.startswith("/run/secrets"):
+        db_server = db_server_url.replace("postgresql://", "")
+        if db_user := self._worker_settings.get("DB_USER", sql_user):
+            if db_password is None:
+                raise AttributeError(
+                    "'Settings' object has no attribute 'DB_PASSWORD'"
+                    "'DB_PASSWORD' is required when using 'DB_USER'"
+                )
+            if db_password.startswith("/run/secrets"):
                 try:
-                    with open(self._worker_settings.SQL_PASSWORD) as f:
-                        sql_password = f.read().rstrip("\n")
+                    with open(db_password) as f:
+                        db_password = f.read().rstrip("\n")
                 except OSError as err:
                     logging.error(str(err))
                     raise err
-            else:
-                sql_password = self._worker_settings.SQL_PASSWORD
-            settings.SQL = rstuf_db(
-                f"postgresql://{sql_user}:{sql_password}@{sql_server}"
-            )
+
+            db_uri = f"postgresql://{db_user}:{db_password}@{db_server}"
         else:
-            settings.SQL = rstuf_db(f"postgresql://{sql_server}")
+            db_uri = f"postgresql://{db_server}"
 
+        logging.info(f"DB URI: {db_uri}")
+        settings.SQL = rstuf_db(db_uri)
         #
         # Backends
         #

tests/unit/tuf_repository_service_worker/test_repository.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023 Repository Service for TUF Contributors
+# SPDX-FileCopyrightText: 2023-2024 Repository Service for TUF Contributors
 # SPDX-FileCopyrightText: 2022-2023 VMware Inc
 #
 # SPDX-License-Identifier: MIT
@@ -181,9 +181,9 @@ def test_write_repository_settings(self, monkeypatch, test_repo):
         ]
 
     def test_refresh_settings_with_sql_user_password(self, test_repo):
-        test_repo._worker_settings.SQL_SERVER = "fake-sql:5433"
-        test_repo._worker_settings.SQL_USER = "psql"
-        test_repo._worker_settings.SQL_PASSWORD = "psqlpass"
+        test_repo._worker_settings.DB_SERVER = "fake-sql:5433"
+        test_repo._worker_settings.DB_USER = "psql"
+        test_repo._worker_settings.DB_PASSWORD = "psqlpass"
         fake_sql = pretend.stub()
         repository.rstuf_db = pretend.call_recorder(lambda *a: fake_sql)
 
@@ -195,20 +195,20 @@ def test_refresh_settings_with_sql_user_password(self, test_repo):
         ]
 
     def test_refresh_settings_with_sql_user_missing_password(self, test_repo):
-        test_repo._worker_settings.SQL_SERVER = "fake-sql:5433"
-        test_repo._worker_settings.SQL_USER = "psql"
+        test_repo._worker_settings.DB_SERVER = "fake-sql:5433"
+        test_repo._worker_settings.DB_USER = "psql"
 
         with pytest.raises(AttributeError) as e:
             test_repo.refresh_settings()
 
-        assert "'Settings' object has no attribute 'SQL_PASSWORD'" in str(e)
+        assert "'Settings' object has no attribute 'DB_PASSWORD'" in str(e)
 
     def test_refresh_settings_with_sql_user_password_secrets(
         self, test_repo, monkeypatch
     ):
-        test_repo._worker_settings.SQL_SERVER = "fake-sql:5433"
-        test_repo._worker_settings.SQL_USER = "psql"
-        test_repo._worker_settings.SQL_PASSWORD = "/run/secrets/SQL_PASSWORD"
+        test_repo._worker_settings.DB_SERVER = "fake-sql:5433"
+        test_repo._worker_settings.DB_USER = "psql"
+        test_repo._worker_settings.DB_PASSWORD = "/run/secrets/DB_PASSWORD"
         fake_data = pretend.stub(
             read=pretend.call_recorder(lambda: "psqlpass\n")
         )
@@ -234,9 +234,9 @@ def test_refresh_settings_with_sql_user_password_secrets_OSError(
         self, test_repo, monkeypatch, caplog
     ):
         caplog.set_level(repository.logging.ERROR)
-        test_repo._worker_settings.SQL_SERVER = "fake-sql:5433"
-        test_repo._worker_settings.SQL_USER = "psql"
-        test_repo._worker_settings.SQL_PASSWORD = "/run/secrets/SQL_PASSWORD"
+        test_repo._worker_settings.DB_SERVER = "fake-sql:5433"
+        test_repo._worker_settings.DB_USER = "psql"
+        test_repo._worker_settings.DB_PASSWORD = "/run/secrets/DB_PASSWORD"
         monkeypatch.setitem(
             repository.__builtins__,
             "open",
@@ -249,6 +249,18 @@ def test_refresh_settings_with_sql_user_password_secrets_OSError(
         assert "No permission /run/secrets/*" in str(e)
         assert "No permission /run/secrets/*" == caplog.messages[0]
 
+    def test_refresh_settings_with_deprecated_sql(
+        self, test_repo, monkeypatch, caplog
+    ):
+        caplog.set_level(repository.logging.WARNING)
+        test_repo._worker_settings.SQL_SERVER = "fake-sql:5433"
+
+        test_repo.refresh_settings()
+        assert (
+            "Using RSTUF_SQL_* environment variables is deprecated. "
+            "Use RSTUF_DB_* instead."
+        ) in caplog.messages
+
     def test__sign(self, test_repo, monkeypatch):
         fake_key_dict = {"keyval": "foo", "keyid": "keyid"}
         fake_settings = pretend.stub(

tox.ini

@@ -9,7 +9,7 @@ setenv =
     RSTUF_WORKER_ID = "test"
     RSTUF_BROKER_SERVER = fakeserver
     RSTUF_REDIS_SERVER = redis://fake-redis
-    RSTUF_SQL_SERVER = postgresql://fake-sql
+    RSTUF_DB_SERVER = postgresql://fake-sql
     RSTUF_STORAGE_BACKEND = LocalStorage
     RSTUF_LOCAL_STORAGE_BACKEND_PATH = ./data-test/s
     DATA_DIR = ./data-test