Introduced new soa key for rotating database passwords

k8s_itests/deployments/kind/cluster-devbox.yaml

@@ -7,6 +7,14 @@ nodes:
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc

paasta_tools/kubernetes/bin/paasta_secrets_sync.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 import base64
+import contextlib
 import hashlib
 import json
 import logging
@@ -24,6 +25,7 @@
 from functools import partial
 from typing import Callable
 from typing import Dict
+from typing import Generator
 from typing import List
 from typing import Optional
 from typing import Set
@@ -98,7 +100,13 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--secret-type",
-        choices=["all", "paasta-secret", "boto-key", "crypto-key"],
+        choices=[
+            "all",
+            "paasta-secret",
+            "boto-key",
+            "crypto-key",
+            "datastore-credentials",
+        ],
         default="all",
         type=str,
         help="Define which type of secret to add/update. Default is 'all'",
@@ -107,6 +115,26 @@ def parse_args() -> argparse.Namespace:
     return args
 
 
+@contextlib.contextmanager
+def set_temporary_environment_variables(
+    **environ: Dict[str, str]
+) -> Generator[None, None, None]:
+    """
+    *Note the return value means "yields None, takes None, and when finished, returns None"*
+
+    Modifies the os.environ variable then yields this temporary state. Resets it when finished.
+
+    :param environ: Environment variables to set
+    """
+    old_environ = dict(os.environ)  # ensure we're storing a copy
+    os.environ.update(environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+
 def main() -> None:
     args = parse_args()
     if args.verbose:
@@ -259,7 +287,9 @@ def sync_all_secrets(
     vault_cluster_config: Dict[str, str],
     soa_dir: str,
     vault_token_file: str,
-    secret_type: Literal["all", "paasta-secret", "crypto-key", "boto-key"] = "all",
+    secret_type: Literal[
+        "all", "paasta-secret", "crypto-key", "boto-key", "datastore-credentials"
+    ] = "all",
     overwrite_namespace: Optional[str] = None,
 ) -> bool:
     results = []
@@ -313,12 +343,26 @@ def sync_all_secrets(
             )
         )
 
+        sync_service_secrets["datastore-credentials"].append(
+            partial(
+                sync_datastore_credentials,
+                kube_client=kube_client,
+                cluster=cluster,
+                service=service,
+                secret_provider_name=secret_provider_name,
+                vault_cluster_config=vault_cluster_config,
+                soa_dir=soa_dir,
+                vault_token_file=vault_token_file,
+            )
+        )
+
         if secret_type == "all":
             results.append(
                 all(sync() for sync in sync_service_secrets["paasta-secret"])
             )
             results.append(all(sync() for sync in sync_service_secrets["boto-key"]))
             results.append(all(sync() for sync in sync_service_secrets["crypto-key"]))
+            # note that since datastore-credentials are in a different vault, they're not synced as part of 'all'
         else:
             results.append(all(sync() for sync in sync_service_secrets[secret_type]))
 
@@ -396,6 +440,88 @@ def sync_secrets(
     return True
 
 
+def sync_datastore_credentials(
+    kube_client: KubeClient,
+    cluster: str,
+    service: str,
+    secret_provider_name: str,
+    vault_cluster_config: Dict[str, str],
+    soa_dir: str,
+    vault_token_file: str,
+) -> bool:
+    config_loader = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+    system_paasta_config = load_system_paasta_config()
+    datastore_credentials_vault_overrides = (
+        system_paasta_config.get_datastore_credentials_vault_overrides()
+    )
+
+    for instance_config in config_loader.instance_configs(
+        cluster=cluster, instance_type_class=KubernetesDeploymentConfig
+    ):
+        datastore_credentials = instance_config.get_datastore_credentials()
+        with set_temporary_environment_variables(
+            **datastore_credentials_vault_overrides
+        ):
+            # expects VAULT_ADDR_OVERRIDE, VAULT_CA_OVERRIDE, and VAULT_TOKEN_OVERRIDE to be set
+            # in order to use a custom vault shard. overriden temporarily in this context
+            provider = get_secret_provider(
+                secret_provider_name=secret_provider_name,
+                soa_dir=soa_dir,
+                service_name=service,
+                cluster_names=[cluster],
+                # overridden by env variables but still needed here for spec validation
+                secret_provider_kwargs={
+                    "vault_cluster_config": vault_cluster_config,
+                    "vault_auth_method": "token",
+                    "vault_token_file": vault_token_file,
+                },
+            )
+
+        for datastore, credentials in datastore_credentials.items():
+            for credential in credentials:
+                vault_path = f"secrets/datastore/{datastore}/{credential}"
+                secrets = provider.get_data_from_vault_path(vault_path)
+                if not secrets:
+                    # no secrets found at this path. skip syncing
+                    log.debug(
+                        f"Warning: no secrets found at requested path {vault_path}."
+                    )
+                    continue
+
+                # decrypt and save in secret_data
+                vault_key_path = get_vault_key_secret_name(vault_path)
+
+                # kubernetes expects data to be base64 encoded binary in utf-8 when put into secret maps
+                # may look like:
+                # {'master': {'passwd': '****', 'user': 'v-approle-mysql-serv-nVcYexH95A2'}, 'reporting': {'passwd': '****', 'user': 'v-approle-mysql-serv-GgCpRIh9Ut7'}, 'slave': {'passwd': '****', 'user': 'v-approle-mysql-serv-PzjPwqNMbqu'}
+                secret_data = {}
+                secret_data[vault_key_path] = base64.b64encode(
+                    json.dumps(secrets).encode("utf-8")
+                ).decode("utf-8")
+
+                if not secret_data:
+                    continue
+
+                # delay between k8s updates
+                time.sleep(0.3)
+
+                create_or_update_k8s_secret(
+                    service=service,
+                    signature_name=instance_config.get_datastore_credentials_signature_name(
+                        datastore=datastore, credential=credential
+                    ),
+                    secret_name=instance_config.get_datastore_credentials_secret_name(
+                        datastore=datastore, credential=credential
+                    ),
+                    get_secret_data=(lambda: secret_data),
+                    secret_signature=_get_dict_signature(secret_data),
+                    kube_client=kube_client,
+                    namespace=instance_config.get_namespace(),
+                )
+
+    return True
+
+
 def sync_crypto_secrets(
     kube_client: KubeClient,
     cluster: str,

paasta_tools/kubernetes_tools.py

@@ -1516,8 +1516,70 @@ def get_pod_volumes(
         if crypto_volume:
             pod_volumes.append(crypto_volume)
 
+        datastore_credentials_secrets_volumes = (
+            self.get_datastore_credentials_secrets_volumes()
+        )
+        for volume in datastore_credentials_secrets_volumes:
+            pod_volumes.append(volume)
+
         return pod_volumes
 
+    def get_datastore_credentials(self) -> Dict:
+        datastore_credentials = self.config_dict.get("datastore_credentials", {})
+        return datastore_credentials
+
+    def get_datastore_credentials_secret_name(
+        self, datastore: str, credential: str
+    ) -> str:
+        return _get_secret_name(
+            self.get_namespace(),
+            "datastore-credentials",
+            self.get_service(),
+            f"{datastore}-{credential}",
+        )
+
+    def get_datastore_credentials_volume_name(
+        self, datastore: str, credential: str
+    ) -> str:
+        return limit_size_with_hash(
+            name=f"datastore-credentials-volume-{datastore}-{credential}-{self.get_sanitised_service_name()}".replace(
+                "_", "--"
+            )
+        )
+
+    def get_datastore_credentials_secrets_volumes(self) -> List[V1Volume]:
+        """
+        For each credential in the soaconfig, find the corresponding secret and volume names
+        and create the associated volumes, one per credential
+        """
+        datastore_credentials = self.get_datastore_credentials()
+        if not datastore_credentials:
+            return []
+
+        result = []
+        for datastore, credentials in datastore_credentials.items():
+            for credential in credentials:
+                volume_name = self.get_datastore_credentials_volume_name(
+                    datastore=datastore, credential=credential
+                )
+                secret_name = self.get_datastore_credentials_secret_name(
+                    datastore=datastore, credential=credential
+                )
+
+                # map the entirety of this secret to the volume name
+                result.append(
+                    V1Volume(
+                        name=volume_name,
+                        secret=V1SecretVolumeSource(
+                            secret_name=secret_name,
+                            default_mode=mode_to_int("0444"),
+                            optional=False,
+                        ),
+                        # note: no 'items' key here to map all secret key/value pairs to files of the same name
+                    )
+                )
+        return result
+
     def get_boto_volume(self) -> Optional[V1Volume]:
         required_boto_keys = self.config_dict.get("boto_keys", [])
         service_name = self.get_sanitised_deployment_name()
@@ -1663,6 +1725,18 @@ def get_volume_mounts(
                         break
                 volume_mounts.append(mount)
 
+        datastore_credentials = self.config_dict.get("datastore_credentials", {})
+        for datastore, credentials in datastore_credentials.items():
+            for credential in credentials:
+                mount = V1VolumeMount(
+                    mount_path=f"/datastore/{datastore}/{credential}",
+                    name=self.get_datastore_credentials_volume_name(
+                        datastore=datastore, credential=credential
+                    ),
+                    read_only=True,
+                )
+                volume_mounts.append(mount)
+
         return volume_mounts
 
     def get_boto_secret_name(self) -> str:
@@ -1694,6 +1768,16 @@ def get_crypto_secret_signature_name(self) -> str:
             self.get_namespace(), "crypto-key", self.get_service(), self.get_instance()
         )
 
+    def get_datastore_credentials_signature_name(
+        self, datastore: str, credential: str
+    ) -> str:
+        return _get_secret_signature_name(
+            self.get_namespace(),
+            "datastore-credentials",
+            self.get_service(),
+            key_name=f"{datastore}-{credential}",
+        )
+
     def get_boto_secret_hash(self) -> Optional[str]:
         return get_secret_signature(
             kube_client=KubeClient(),

paasta_tools/secret_providers/vault.py

@@ -155,6 +155,22 @@ def get_secret_signature_from_data(self, data: Mapping[str, Any]) -> Optional[st
         else:
             return None
 
+    def get_data_from_vault_path(self, path: str) -> Optional[dict]:
+        # clients.read returns None if not set
+        # if it is set, it returns an object with { **metadata, data: {} }
+        # eg lease_id, request_id, etc. we only care about 'data' here
+        entry = self.clients[self.ecosystems[0]].read(path)
+
+        # returns one of 3 things:
+        #   entry -> could be None
+        #   entry["data"] -> could be an object with content
+        #   entry["data"] -> could be empty (ie no secrets)
+        # all are plausible and valid scenarios that give different information about the path
+
+        if entry is not None:
+            return entry.get("data", {})
+        return None
+
     def get_key_versions(
         self,
         key_name: str,

paasta_tools/utils.py

@@ -2562,6 +2562,13 @@ def get_topology_spread_constraints(self) -> List[Dict[str, Any]]:
         """List of TopologySpreadConstraints that will be applied to all Pods in the cluster"""
         return self.config_dict.get("topology_spread_constraints", [])
 
+    def get_datastore_credentials_vault_overrides(self) -> Dict:
+        """In order to use different Vault shards, vault-tools allows you to override
+        environment variables (CA, token file, and URL). DB credentials are stored in
+        a different shard to minimize the impact on the core Vault shard (which has
+        size restrictions derived from Zookeeper limitations)."""
+        return self.config_dict.get("datastore_credentials_vault_env_overrides", {})
+
     def get_vault_environment(self) -> Optional[str]:
         """Get the environment name for the vault cluster
         This must match the environment keys in the secret json files