Introduced new soa key for rotating database passwords

k8s_itests/deployments/kind/cluster-devbox.yaml

@@ -7,6 +7,14 @@ nodes:
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc
   - role: worker
     labels:
       yelp.com/pool: default
+    # this mount is needed for services which rely on /nail/etc/{ecosystem,runtime}
+    extraMounts:
+      - hostPath: /nail/etc
+        containerPath: /nail/etc

paasta_tools/cli/schemas/kubernetes_schema.json

@@ -486,6 +486,20 @@
                         }
                     }
                 },
+                "datastore_credentials": {
+                    "type": "object",
+                    "additionalProperties": true,
+                    "properties": {
+                        "mysql": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                                "pattern": "^[a-zA-Z0-9_-]+$"
+                            },
+                            "uniqueItems": true
+                        }
+                    }
+                },
                 "extra_volumes": {
                     "type": "array",
                     "items": {

paasta_tools/kubernetes/bin/paasta_secrets_sync.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 import base64
+import contextlib
 import hashlib
 import json
 import logging
@@ -24,6 +25,7 @@
 from functools import partial
 from typing import Callable
 from typing import Dict
+from typing import Generator
 from typing import List
 from typing import Optional
 from typing import Set
@@ -98,7 +100,13 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument(
         "--secret-type",
-        choices=["all", "paasta-secret", "boto-key", "crypto-key"],
+        choices=[
+            "all",
+            "paasta-secret",
+            "boto-key",
+            "crypto-key",
+            "datastore-credentials",
+        ],
         default="all",
         type=str,
         help="Define which type of secret to add/update. Default is 'all'",
@@ -107,6 +115,26 @@ def parse_args() -> argparse.Namespace:
     return args
 
 
+@contextlib.contextmanager
+def set_temporary_environment_variables(
+    **environ: Dict[str, str]
+) -> Generator[None, None, None]:
+    """
+    *Note the return value means "yields None, takes None, and when finished, returns None"*
+
+    Modifies the os.environ variable then yields this temporary state. Resets it when finished.
+
+    :param environ: Environment variables to set
+    """
+    old_environ = dict(os.environ)  # ensure we're storing a copy
+    os.environ.update(environ)
+    try:
+        yield
+    finally:
+        os.environ.clear()
+        os.environ.update(old_environ)
+
+
 def main() -> None:
     args = parse_args()
     if args.verbose:
@@ -259,7 +287,9 @@ def sync_all_secrets(
     vault_cluster_config: Dict[str, str],
     soa_dir: str,
     vault_token_file: str,
-    secret_type: Literal["all", "paasta-secret", "crypto-key", "boto-key"] = "all",
+    secret_type: Literal[
+        "all", "paasta-secret", "crypto-key", "boto-key", "datastore-credentials"
+    ] = "all",
     overwrite_namespace: Optional[str] = None,
 ) -> bool:
     results = []
@@ -313,12 +343,27 @@ def sync_all_secrets(
             )
         )
 
+        sync_service_secrets["datastore-credentials"].append(
+            partial(
+                sync_datastore_credentials,
+                kube_client=kube_client,
+                cluster=cluster,
+                service=service,
+                secret_provider_name=secret_provider_name,
+                vault_cluster_config=vault_cluster_config,
+                soa_dir=soa_dir,
+                vault_token_file=vault_token_file,
+                overwrite_namespace=overwrite_namespace,
+            )
+        )
+
         if secret_type == "all":
             results.append(
                 all(sync() for sync in sync_service_secrets["paasta-secret"])
             )
             results.append(all(sync() for sync in sync_service_secrets["boto-key"]))
             results.append(all(sync() for sync in sync_service_secrets["crypto-key"]))
+            # note that since datastore-credentials are in a different vault, they're not synced as part of 'all'
         else:
             results.append(all(sync() for sync in sync_service_secrets[secret_type]))
 
@@ -396,6 +441,88 @@ def sync_secrets(
     return True
 
 
+def sync_datastore_credentials(
+    kube_client: KubeClient,
+    cluster: str,
+    service: str,
+    secret_provider_name: str,
+    vault_cluster_config: Dict[str, str],
+    soa_dir: str,
+    vault_token_file: str,
+    overwrite_namespace: Optional[str] = None,
+) -> bool:
+    """
+    Map all the passwords requested for this service-instance to a single Kubernetes Secret store.
+    Volume mounts will then map the associated secrets to their associated mount paths.
+    """
+    config_loader = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+    system_paasta_config = load_system_paasta_config()
+    datastore_credentials_vault_overrides = (
+        system_paasta_config.get_datastore_credentials_vault_overrides()
+    )
+
+    for instance_config in config_loader.instance_configs(
+        cluster=cluster, instance_type_class=KubernetesDeploymentConfig
+    ):
+        namespace = (
+            overwrite_namespace
+            if overwrite_namespace is not None
+            else instance_config.get_namespace()
+        )
+        datastore_credentials = instance_config.get_datastore_credentials()
+        with set_temporary_environment_variables(
+            **datastore_credentials_vault_overrides
+        ):
+            # expects VAULT_ADDR_OVERRIDE, VAULT_CA_OVERRIDE, and VAULT_TOKEN_OVERRIDE to be set
+            # in order to use a custom vault shard. overriden temporarily in this context
+            provider = get_secret_provider(
+                secret_provider_name=secret_provider_name,
+                soa_dir=soa_dir,
+                service_name=service,
+                cluster_names=[cluster],
+                # overridden by env variables but still needed here for spec validation
+                secret_provider_kwargs={
+                    "vault_cluster_config": vault_cluster_config,
+                    "vault_auth_method": "token",
+                    "vault_token_file": vault_token_file,
+                },
+            )
+
+        secret_data = {}
+        for datastore, credentials in datastore_credentials.items():
+            for credential in credentials:
+                vault_path = f"secrets/datastore/{datastore}/{credential}"
+                secrets = provider.get_data_from_vault_path(vault_path)
+                if not secrets:
+                    # no secrets found at this path. skip syncing
+                    log.debug(
+                        f"Warning: no secrets found at requested path {vault_path}."
+                    )
+                    continue
+
+                # decrypt and save in secret_data
+                vault_key_path = get_vault_key_secret_name(vault_path)
+
+                # kubernetes expects data to be base64 encoded binary in utf-8 when put into secret maps
+                # may look like:
+                # {'master': {'passwd': '****', 'user': 'v-approle-mysql-serv-nVcYexH95A2'}, 'reporting': {'passwd': '****', 'user': 'v-approle-mysql-serv-GgCpRIh9Ut7'}, 'slave': {'passwd': '****', 'user': 'v-approle-mysql-serv-PzjPwqNMbqu'}
+                secret_data[vault_key_path] = base64.b64encode(
+                    json.dumps(secrets).encode("utf-8")
+                ).decode("utf-8")
+
+        create_or_update_k8s_secret(
+            service=service,
+            signature_name=instance_config.get_datastore_credentials_signature_name(),
+            secret_name=instance_config.get_datastore_credentials_secret_name(),
+            get_secret_data=(lambda: secret_data),
+            secret_signature=_get_dict_signature(secret_data),
+            kube_client=kube_client,
+            namespace=namespace,
+        )
+
+    return True
+
+
 def sync_crypto_secrets(
     kube_client: KubeClient,
     cluster: str,
@@ -447,8 +574,6 @@ def sync_crypto_secrets(
         if not secret_data:
             continue
 
-        # In order to prevent slamming the k8s API, add some artificial delay here
-        time.sleep(0.3)
         create_or_update_k8s_secret(
             service=service,
             signature_name=instance_config.get_crypto_secret_signature_name(),
@@ -496,8 +621,6 @@ def sync_boto_secrets(
         if not secret_data:
             continue
 
-        # In order to prevent slamming the k8s API, add some artificial delay here
-        time.sleep(0.3)
         create_or_update_k8s_secret(
             service=service,
             signature_name=instance_config.get_boto_secret_signature_name(),
@@ -528,6 +651,9 @@ def create_or_update_k8s_secret(
     """
     :param get_secret_data: is a function to postpone fetching data in order to reduce service load, e.g. Vault API
     """
+    # In order to prevent slamming the k8s API, add some artificial delay here
+    time.sleep(0.3)
+
     kubernetes_signature = get_secret_signature(
         kube_client=kube_client,
         signature_name=signature_name,

paasta_tools/kubernetes_tools.py

@@ -282,6 +282,10 @@ class KubeWeightedAffinityCondition(KubeAffinityCondition):
     weight: int
 
 
+class DatastoreCredentialsConfig(TypedDict, total=False):
+    mysql: List[str]
+
+
 def _set_disrupted_pods(self: Any, disrupted_pods: Mapping[str, datetime]) -> None:
     """Private function used to patch the setter for V1beta1PodDisruptionBudgetStatus.
     Can be removed once https://github.com/kubernetes-client/python/issues/466 is resolved
@@ -1516,8 +1520,69 @@ def get_pod_volumes(
         if crypto_volume:
             pod_volumes.append(crypto_volume)
 
+        datastore_credentials_secrets_volume = (
+            self.get_datastore_credentials_secrets_volume()
+        )
+        if datastore_credentials_secrets_volume:
+            pod_volumes.append(datastore_credentials_secrets_volume)
+
         return pod_volumes
 
+    def get_datastore_credentials(self) -> DatastoreCredentialsConfig:
+        datastore_credentials = self.config_dict.get("datastore_credentials", {})
+        return datastore_credentials
+
+    def get_datastore_credentials_secret_name(self) -> str:
+        return _get_secret_name(
+            self.get_namespace(),
+            "datastore-credentials",
+            self.get_service(),
+            self.get_instance(),
+        )
+
+    def get_datastore_credentials_volume_name(self) -> str:
+        """
+        Volume names must abide to DNS mappings of 63 chars or less, so we limit it here and replace _ with --.
+        """
+        prefix = "datastore-credentials-volume"
+        service_name = self.get_sanitised_service_name()
+        instance_name = self.get_sanitised_instance_name()
+        return limit_size_with_hash(
+            name=f"{prefix}-{service_name}-{instance_name}".replace("_", "--")
+        )
+
+    def get_datastore_credentials_secrets_volume(self) -> List[V1Volume]:
+        """
+        All credentials are stored in 1 Kubernetes Secret, which are mapped on an item->path
+        structure to /datastore/<datastore>/<credential>/<password file>.
+        """
+        datastore_credentials = self.get_datastore_credentials()
+        if not datastore_credentials:
+            return None
+
+        secrets_with_custom_mountpaths = []
+        for datastore, credentials in datastore_credentials.items():
+            for credential in credentials:
+                secrets_with_custom_mountpaths.append(
+                    {
+                        "key": get_vault_key_secret_name(
+                            f"secrets/datastore/{datastore}/{credential}"
+                        ),
+                        "mode": mode_to_int("0444"),
+                        "path": f"{datastore}/{credential}/credentials",
+                    }
+                )
+
+        return V1Volume(
+            name=self.get_datastore_credentials_volume_name(),
+            secret=V1SecretVolumeSource(
+                secret_name=self.get_datastore_credentials_secret_name(),
+                default_mode=mode_to_int("0444"),
+                items=secrets_with_custom_mountpaths,
+                optional=False,
+            ),
+        )
+
     def get_boto_volume(self) -> Optional[V1Volume]:
         required_boto_keys = self.config_dict.get("boto_keys", [])
         service_name = self.get_sanitised_deployment_name()
@@ -1663,6 +1728,16 @@ def get_volume_mounts(
                         break
                 volume_mounts.append(mount)
 
+        datastore_credentials = self.config_dict.get("datastore_credentials", {})
+        if datastore_credentials:
+            volume_mounts.append(
+                V1VolumeMount(
+                    mount_path=f"/datastore",
+                    name=self.get_datastore_credentials_volume_name(),
+                    read_only=True,
+                )
+            )
+
         return volume_mounts
 
     def get_boto_secret_name(self) -> str:
@@ -1694,6 +1769,18 @@ def get_crypto_secret_signature_name(self) -> str:
             self.get_namespace(), "crypto-key", self.get_service(), self.get_instance()
         )
 
+    def get_datastore_credentials_signature_name(self) -> str:
+        """
+        All datastore credentials are stored in a single Kubernetes secret, so they share a name
+        """
+        return _get_secret_signature_name(
+            self.get_namespace(),
+            "datastore-credentials",
+            self.get_service(),
+            # key is on instances, which get their own configurations
+            key_name=self.get_instance(),
+        )
+
     def get_boto_secret_hash(self) -> Optional[str]:
         return get_secret_signature(
             kube_client=KubeClient(),

paasta_tools/secret_providers/vault.py

@@ -155,6 +155,31 @@ def get_secret_signature_from_data(self, data: Mapping[str, Any]) -> Optional[st
         else:
             return None
 
+    def get_data_from_vault_path(self, path: str) -> Optional[Dict[str, str]]:
+        # clients.read returns None if not set
+        # if it is set, it returns an object with { **metadata, data: {} }
+        # eg lease_id, request_id, etc. we only care about 'data' here
+
+        client = self.clients[self.ecosystems[0]]
+
+        # there is a chance client is None (ie if the connection is invalid)
+        if client is None:
+            raise Exception(
+                "possible Vault misconfiguration as client is not available"
+            )
+
+        entry = client.read(path)
+
+        # returns one of 3 things:
+        #   entry -> could be None
+        #   entry["data"] -> could be an object with content
+        #   entry["data"] -> could be empty (ie no secrets)
+        # all are plausible and valid scenarios that give different information about the path
+
+        if entry is not None:
+            return entry.get("data", {})
+        return None
+
     def get_key_versions(
         self,
         key_name: str,