Prepare for securesystemslib v1.0.0 (#507)

* Update securesystemslib 1.0.0 (WIP) * Update securesystemslib 1.0.0 in requirements*.txt files -> requires pinning a dev version of tuf TODO: - adopt in Pipfile (I tried, but `pipenv lock` was taking way too long for my taste) - update tuf when theupdateframework/python-tuf#2617 is released * Remove local keyvault service, which makes heavy use of legacy securesystemslib interfaces, which are no longer available in 1.0.0. TODO: - adopt in docs, config, etc - consider removing obsolete IKeyVault * Remove keyvault initialisation in MetadatRepository, which would try to load local key vault in tests, given the current configs (see TODO above) * Adopt removal in tests, just enough, so that they pass. TODO: - check if the tests still make sense * Drop registration of CryptoSigner and use its new uri scheme "file2" in SignerStore. "file2" can be used like "file", but only for non-encrypted key files, which is all we care for in the worker. "file2" can also be used like "fn" from the custom "FileNameSigner", i.e. with a directory specified via envvar. TODO: - consider only using "file2" and dropping the custom "FileNameSigner" (or only using it to ovverride the scheme name and the envvar name) Signed-off-by: Lukas Puehringer <[email protected]> * chore: dependencies and ci/cd Signed-off-by: Kairo Araujo <[email protected]> * fixup! Update securesystemslib 1.0.0 (WIP) Signed-off-by: Kairo Araujo <[email protected]> * docs: update related docs about IKeyVault removal Signed-off-by: Kairo Araujo <[email protected]> * fixup! fixup! Update securesystemslib 1.0.0 (WIP) * fix: update the dependencies, including securesystemslib Signed-off-by: Kairo Araujo <[email protected]> --------- Signed-off-by: Lukas Puehringer <[email protected]> Signed-off-by: Kairo Araujo <[email protected]> Co-authored-by: Kairo Araujo <[email protected]>
repository-service-tuf · Jul 26, 2024 · a4c6e73 · a4c6e73
1 parent 1285654
commit a4c6e73
Show file tree

Hide file tree

Showing 24 changed files with 439 additions and 1,861 deletions.
diff --git a/Pipfile b/Pipfile
@@ -9,13 +9,13 @@ redis = "*"
 tuf = "*"
 dynaconf = {extras = ["ini"], version = "*"}
 supervisor = "*"
-securesystemslib = {extras = ["pynacl", "crypto"], version = ">-=0.31.0,<1.0.0"}
+securesystemslib = "*"
 sqlalchemy = "*"
 psycopg2 = "*"
 alembic = "*"
 pydantic = "*"
 celery = "*"
-boto3 = "==1.34.93"
+boto3 = "*"
 awswrangler = "*"
 
 [dev-packages]

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/docs/diagrams/repository-service-tuf-worker-C2.puml b/docs/diagrams/repository-service-tuf-worker-C2.puml
@@ -56,14 +56,8 @@ Container_Boundary(repository_storage, "Storage Service"){
     Container_Ext(aws_s3, "AWS S3", "Storage", $tags="aws_s3") #Grey
     Container_Ext(azure_blob, "Azure Blob", "Storage", $tags="azure_blob") #Grey
 }
-Container_Boundary(key_storage, "Key Vault Service"){
-    Container_Ext(key_fs, "Filesystem", "Storage", $tags="storage_service") #Grey
-    Container_Ext(aws_kms, "AWS KMS", "Storage", $tags="aws_kms") #Grey
-    Container_Ext(azure_kv, "Azure Key Vault", "Storage", $tags="azure_kv") #Grey
-}
 Rel_D(broker, repository_service_tuf_worker, "Consumer", "Tasks")
 Rel_U(repository_service_tuf_worker, broker, "Publisher", "Tasks Results")
-Rel_U(key_storage, repository_service_tuf_worker, "Read", "Online Keys")
 BiRel_U(repository_service_tuf_worker, redis, "Repository Configuration", "Consumer")
 BiRel_U(repository_service_tuf_worker, sql, "TargetsFiles, TargetsRoles", "")
 BiRel_R(repository_service_tuf_worker, data_dir, "Write/Read", "Service settings")

diff --git a/docs/diagrams/repository-service-tuf-worker-C3.puml b/docs/diagrams/repository-service-tuf-worker-C3.puml
@@ -20,7 +20,6 @@
 
 
 AddContainerTag("webui", $sprite="application_server", $legendText="Web UI Interface", $bgColor=Gray)
-AddContainerTag("key_service", $sprite="lock_with_key_security", $legendText="Key Service", $bgColor=Gray)
 AddContainerTag("storage_service", $sprite="file_server", $legendText="Storage Service", $bgColor=Gray)
 AddContainerTag("rest_api", $sprite="web_server", $legendText="Repository REST API")
 AddContainerTag("repository_service_tuf_worker", $sprite="service_application", $legendText="Repository Metadata Worker")
@@ -54,12 +53,8 @@ System_Boundary(repository_service_tuf_worker, "Repository Service for TUF Worke
         }
         Container_Boundary(interfaces, "interfaces") #CornflowerBlue {
             Container(IStorage, "Storage Interface")
-            Container(IKeyVault, "Key Vault Interface")
         }
         Container_Boundary(services, "services") #DeepSkyBlue{
-            Container_Boundary(keyvault, "keyvault") #LightSteelBlue {
-                Container(LocalKeyVault, "LocalKeyVault", "File System")
-            }
             Container_Boundary(storage, "storage") #LightSteelBlue {
                 Container(LocalStorage, "LocalStorage", "File System")
             }
@@ -73,14 +68,10 @@ Container_Ext(broker, "Broker", "RabbitMQ, Redis, etc", $tags="queue") #Grey
 Container_Ext(redis, "Redis", "Redis Server", $tags="queue") #Grey
 Container_Ext(sql, "SQL", "SQL Server", $tags="sql_db") #Grey
 Container_Ext(ext_storage, "Metadata Storage", "specific technology", $tags="key_service") #Grey
-Container_Ext(ext_key_storage, "Key Vault Storage", "specific technology", $tags="storage_service") #Grey
 
 Rel(LocalStorage, ext_storage, " ")
-Rel(LocalKeyVault, ext_key_storage, " ")
-Rel(IKeyVault, LocalKeyVault, "uses")
 Rel(IStorage, LocalStorage, "uses")
 Rel(metadata_repository, IStorage, " ")
-Rel(metadata_repository, IKeyVault, " ")
 Rel(metadata_repository, celery, " ")
 Rel(metadata_repository, repository, " ")
 Rel(metadata_repository, dynaconf, " ")

diff --git a/docs/source/_static/repository-service-tuf-worker-C2.png b/docs/source/_static/repository-service-tuf-worker-C2.png
diff --git a/docs/source/_static/repository-service-tuf-worker-C3.png b/docs/source/_static/repository-service-tuf-worker-C3.png
diff --git a/docs/source/devel/index.rst b/docs/source/devel/index.rst
@@ -67,7 +67,7 @@ Bootstrap
     @enduml
 
 Adding/Removing artifacts
------------------------
+-------------------------
 
 As mentioned at the container level, the domain of ``repository-service-tuf-worker``
 (Repository Worker) is managing the TUF Repository Metadata.
@@ -108,9 +108,7 @@ About **Bump Roles** (``bump_online_roles``) that contain online keys is easy.
 These roles have short expiration (defined during repository configuration) and
 must be "bumped" frequently. The implementation in the RepositoryMetadata
 
-**Publish the new Hashed Bins Target Roles** (``publish_artifacts``) is part of the
-solution for the :ref:`Repository Worker scalability, Issue 17
-<devel/known_issues:(Solved) Scalability>`.
+**Publish the new Hashed Bins Target Roles** (``publish_artifacts``)
 
 To understand more, every time the API sends a task to add a new artifact(s), the
 Hashed Bins Roles must be changed to add the new artifact(s), followed by a new
@@ -219,14 +217,6 @@ AWSS3
 1. Initiate the aws development environment ``make run-dev DC=aws``
 
 
-Important issues/problems
-#########################
-
-.. toctree::
-   :maxdepth: 1
-
-   known_issues
-
 Implementation
 ##############
 
@@ -238,5 +228,4 @@ Implementation
    repository_service_tuf_worker.models.targets
    repository_service_tuf_worker.services
    repository_service_tuf_worker.services.storage
-   repository_service_tuf_worker.services.keyvault
    modules