Introduced new soa key for rotating database passwords

k8s_itests/deployments/kind/cluster-devbox.yaml

@@ -7,6 +7,14 @@ nodes:
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc

paasta_tools/cli/schemas/kubernetes_schema.json

@@ -486,6 +486,20 @@
                         }
                     }
                 },
+                "datastore_credentials": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "mysql": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                                "pattern": "^[a-zA-Z0-9_-]+$"
+                            },
+                            "uniqueItems": true
+                        }
+                    }
+                },
                 "extra_volumes": {
                     "type": "array",
                     "items": {

paasta_tools/kubernetes/bin/paasta_secrets_sync.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 import base64
+import contextlib
 import hashlib
 import json
 import logging
@@ -24,7 +25,9 @@
 from functools import partial
 from typing import Callable
 from typing import Dict
+from typing import Generator
 from typing import List
+from typing import Mapping
 from typing import Optional
 from typing import Set
 from typing import Tuple
@@ -99,15 +102,41 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--secret-type",
-        choices=["all", "paasta-secret", "boto-key", "crypto-key"],
+        choices=[
+            "all",
+            "paasta-secret",
+            "boto-key",
+            "crypto-key",
+            "datastore-credentials",
+        ],
         default="all",
         type=str,
-        help="Define which type of secret to add/update. Default is 'all'",
+        help="Define which type of secret to add/update. Default is 'all' (which does not include datastore-credentials)",
     )
     args = parser.parse_args()
     return args
 
 
+@contextlib.contextmanager
+def set_temporary_environment_variables(
+    environ: Mapping[str, str]
+) -> Generator[None, None, None]:
+    """
+    *Note the return value means "yields None, takes None, and when finished, returns None"*
+
+    Modifies the os.environ variable then yields this temporary state. Resets it when finished.
+
+    :param environ: Environment variables to set
+    """
+    old_environ = dict(os.environ)  # ensure we're storing a copy
+    os.environ.update(environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+
 def main() -> None:
     args = parse_args()
     if args.verbose:
@@ -260,7 +289,9 @@ def sync_all_secrets(
     vault_cluster_config: Dict[str, str],
     soa_dir: str,
     vault_token_file: str,
-    secret_type: Literal["all", "paasta-secret", "crypto-key", "boto-key"] = "all",
+    secret_type: Literal[
+        "all", "paasta-secret", "crypto-key", "boto-key", "datastore-credentials"
+    ] = "all",
     overwrite_namespace: Optional[str] = None,
 ) -> bool:
     results = []
@@ -314,12 +345,27 @@ def sync_all_secrets(
             )
         )
 
+        sync_service_secrets["datastore-credentials"].append(
+            partial(
+                sync_datastore_credentials,
+                kube_client=kube_client,
+                cluster=cluster,
+                service=service,
+                secret_provider_name=secret_provider_name,
+                vault_cluster_config=vault_cluster_config,
+                soa_dir=soa_dir,
+                vault_token_file=vault_token_file,
+                overwrite_namespace=overwrite_namespace,
+            )
+        )
+
         if secret_type == "all":
             results.append(
                 all(sync() for sync in sync_service_secrets["paasta-secret"])
             )
             results.append(all(sync() for sync in sync_service_secrets["boto-key"]))
             results.append(all(sync() for sync in sync_service_secrets["crypto-key"]))
+            # note that since datastore-credentials are in a different vault, they're not synced as part of 'all'
         else:
             results.append(all(sync() for sync in sync_service_secrets[secret_type]))
 
@@ -397,6 +443,88 @@ def sync_secrets(
     return True
 
 
+def sync_datastore_credentials(
+    kube_client: KubeClient,
+    cluster: str,
+    service: str,
+    secret_provider_name: str,
+    vault_cluster_config: Dict[str, str],
+    soa_dir: str,
+    vault_token_file: str,
+    overwrite_namespace: Optional[str] = None,
+) -> bool:
+    """
+    Map all the passwords requested for this service-instance to a single Kubernetes Secret store.
+    Volume mounts will then map the associated secrets to their associated mount paths.
+    """
+    config_loader = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+    system_paasta_config = load_system_paasta_config()
+    datastore_credentials_vault_overrides = (
+        system_paasta_config.get_datastore_credentials_vault_overrides()
+    )
+
+    for instance_config in config_loader.instance_configs(
+        cluster=cluster, instance_type_class=KubernetesDeploymentConfig
+    ):
+        namespace = (
+            overwrite_namespace
+            if overwrite_namespace is not None
+            else instance_config.get_namespace()
+        )
+        datastore_credentials = instance_config.get_datastore_credentials()
+        with set_temporary_environment_variables(datastore_credentials_vault_overrides):
+            # expects VAULT_ADDR_OVERRIDE, VAULT_CA_OVERRIDE, and VAULT_TOKEN_OVERRIDE to be set
+            # in order to use a custom vault shard. overriden temporarily in this context
+            provider = get_secret_provider(
+                secret_provider_name=secret_provider_name,
+                soa_dir=soa_dir,
+                service_name=service,
+                cluster_names=[cluster],
+                # overridden by env variables but still needed here for spec validation
+                secret_provider_kwargs={
+                    "vault_cluster_config": vault_cluster_config,
+                    "vault_auth_method": "token",
+                    "vault_token_file": vault_token_file,
+                },
+            )
+
+            secret_data = {}
+            for datastore, credentials in datastore_credentials.items():
+                # mypy loses type hints on '.items' and throws false positives. unfortunately have to type: ignore
+                # https://github.com/python/mypy/issues/7178
+                for credential in credentials:  # type: ignore
+                    vault_path = f"secrets/datastore/{datastore}/{credential}"
+                    secrets = provider.get_data_from_vault_path(vault_path)
+                    if not secrets:
+                        # no secrets found at this path. skip syncing
+                        log.debug(
+                            f"Warning: no secrets found at requested path {vault_path}."
+                        )
+                        continue
+
+                    # decrypt and save in secret_data
+                    vault_key_path = get_vault_key_secret_name(vault_path)
+
+                    # kubernetes expects data to be base64 encoded binary in utf-8 when put into secret maps
+                    # may look like:
+                    # {'master': {'passwd': '****', 'user': 'v-approle-mysql-serv-nVcYexH95A2'}, 'reporting': {'passwd': '****', 'user': 'v-approle-mysql-serv-GgCpRIh9Ut7'}, 'slave': {'passwd': '****', 'user': 'v-approle-mysql-serv-PzjPwqNMbqu'}
+                    secret_data[vault_key_path] = base64.b64encode(
+                        json.dumps(secrets).encode("utf-8")
+                    ).decode("utf-8")
+
+        create_or_update_k8s_secret(
+            service=service,
+            signature_name=instance_config.get_datastore_credentials_signature_name(),
+            secret_name=instance_config.get_datastore_credentials_secret_name(),
+            get_secret_data=(lambda: secret_data),
+            secret_signature=_get_dict_signature(secret_data),
+            kube_client=kube_client,
+            namespace=namespace,
+        )
+
+    return True
+
+
 def sync_crypto_secrets(
     kube_client: KubeClient,
     cluster: str,
@@ -448,8 +576,6 @@ def sync_crypto_secrets(
         if not secret_data:
             continue
 
-        # In order to prevent slamming the k8s API, add some artificial delay here
-        time.sleep(0.3)
         create_or_update_k8s_secret(
             service=service,
             signature_name=instance_config.get_crypto_secret_signature_name(),
@@ -497,8 +623,6 @@ def sync_boto_secrets(
         if not secret_data:
             continue
 
-        # In order to prevent slamming the k8s API, add some artificial delay here
-        time.sleep(0.3)
         create_or_update_k8s_secret(
             service=service,
             signature_name=instance_config.get_boto_secret_signature_name(),
@@ -529,6 +653,9 @@ def create_or_update_k8s_secret(
     """
     :param get_secret_data: is a function to postpone fetching data in order to reduce service load, e.g. Vault API
     """
+    # In order to prevent slamming the k8s API, add some artificial delay here
+    time.sleep(0.3)
+
     kubernetes_signature = get_secret_signature(
         kube_client=kube_client,
         signature_name=signature_name,