From e3bf180b82fdedb4273856f690e17ab16d17c04c Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Wed, 4 Sep 2024 12:44:15 -0700 Subject: [PATCH 1/3] Revert "Fix a test failure" This reverts commit 4872e2ee0aaaf955dec41b8388dbec98fba8f472. --- tests/test_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 47830d1ae9..12b8442fbf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2363,8 +2363,6 @@ def test_validate_service_instance_invalid(): mock_nrtsearcheks_instances = [("service1", "nrtsearcheks")] mock_monkrelaycluster_instances = [("service1", "monkrelays")] mock_vitesscluster_instances = [("service1", "vitesscluster")] - mock_vitesscell_instances = [("service1", "vitesscell")] - mock_vitesskeyspace_instances = [("service1", "vitesskeyspace")] my_service = "service1" my_instance = "main" fake_cluster = "fake_cluster" @@ -2387,8 +2385,6 @@ def test_validate_service_instance_invalid(): mock_nrtsearcheks_instances, mock_monkrelaycluster_instances, mock_vitesscluster_instances, - mock_vitesscell_instances, - mock_vitesskeyspace_instances, ], ): with raises( From 5327ee09e95f15fed53b6b243068cca40c000c96 Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Wed, 4 Sep 2024 12:45:22 -0700 Subject: [PATCH 2/3] Revert "Fix tests and reduce code redundancy" This reverts commit b45e1b75862cbc3cfa714718429380e077f98bb8. --- paasta_tools/vitesscell_tools.py | 79 +++- paasta_tools/vitesscluster_tools.py | 82 ++--- paasta_tools/vitesskeyspace_tools.py | 81 +++- tests/test_vitesscluster_tools.py | 529 +++++++++++++++++++++++++-- 4 files changed, 693 insertions(+), 78 deletions(-) diff --git a/paasta_tools/vitesscell_tools.py b/paasta_tools/vitesscell_tools.py index 0bdb9918f7..25ba088236 100644 --- a/paasta_tools/vitesscell_tools.py +++ b/paasta_tools/vitesscell_tools.py @@ -1,4 +1,6 @@ +import json import logging +import sys from typing import Any from typing import Dict from typing import List @@ -8,12 +10,16 @@ from typing import Union import service_configuration_lib +from kubernetes.client import ApiClient +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict from paasta_tools.kubernetes_tools import sanitised_cr_name +from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 from paasta_tools.utils import deep_merge_dictionaries from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_service_instance_config from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import load_v2_deployments_json @@ -22,8 +28,6 @@ from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom from paasta_tools.vitesscluster_tools import RequestsDict from paasta_tools.vitesscluster_tools import ResourceConfigDict -from paasta_tools.vitesscluster_tools import VitessDeploymentConfig -from paasta_tools.vitesscluster_tools import VitessDeploymentConfigDict log = logging.getLogger(__name__) @@ -63,8 +67,9 @@ class GatewayConfigDict(TypedDict, total=False): annotations: Mapping[str, Any] -class VitessCellConfigDict(VitessDeploymentConfigDict, total=False): +class VitessCellConfigDict(KubernetesDeploymentConfigDict, total=False): name: str + images: Dict[str, str] allCells: List[str] globalLockserver: Dict[str, str] gateway: GatewayConfigDict @@ -133,7 +138,7 @@ class VitessCellInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] -class VitessCellConfig(VitessDeploymentConfig): +class VitessCellConfig(KubernetesDeploymentConfig): config_dict: VitessCellInstanceConfigDict config_filename_prefix = "vitesscell" @@ -156,6 +161,9 @@ def __init__( branch_dict=branch_dict, ) + def get_namespace(self) -> str: + return KUBERNETES_NAMESPACE + def get_images(self) -> Dict[str, str]: vitess_images = self.config_dict.get( "images", load_system_paasta_config().get_vitess_images() @@ -167,6 +175,69 @@ def get_images(self) -> Dict[str, str]: "vttablet": vitess_images["vttablet_image"], } + def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: + # get all K8s container env vars and format their keys to camel case + + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + env = [ + api_client.sanitize_for_serialization(env) + for env in self.get_container_env() + ] + return env + + def get_labels(self) -> Dict[str, str]: + # get default labels from parent class to adhere to paasta contract + docker_url = self.get_docker_url( + system_paasta_config=load_system_paasta_config() + ) + git_sha = get_git_sha_from_dockerurl(docker_url) + labels = self.get_kubernetes_metadata(git_sha=git_sha).labels + if "yelp.com/owner" in labels.keys(): + labels["yelp.com/owner"] = "dre_mysql" + return labels + + def get_annotations(self) -> Mapping[str, Any]: + # get required annotations to be added to the formatted resource before creating or updating custom resource + service_namespace_config = load_service_namespace_config( + service=self.service, namespace=self.get_nerve_namespace() + ) + system_paasta_config = load_system_paasta_config() + has_routable_ip = self.has_routable_ip( + service_namespace_config, system_paasta_config + ) + annotations: Mapping[str, Any] = { + "smartstack_registrations": json.dumps(self.get_registrations()), + "paasta.yelp.com/routable_ip": has_routable_ip, + } + + return annotations + + def get_vitess_node_affinity(self) -> dict: + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) + return node_affinity + + def get_region(self) -> str: + superregion = self.get_cluster() + superregion_to_region_map = ( + load_system_paasta_config().get_superregion_to_region_mapping() + ) + region = None + for superregion_prefix in superregion_to_region_map: + if superregion.startswith(superregion_prefix): + region = superregion.replace( + superregion_prefix, superregion_to_region_map[superregion_prefix] + ) + if region is None: + log.error( + f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" + ) + # Exiting early here since region is needed to fetch secrets from vault + sys.exit(1) + return region + def get_global_lock_server(self) -> Dict[str, str]: zk_address = self.config_dict.get("zk_address") return { diff --git a/paasta_tools/vitesscluster_tools.py b/paasta_tools/vitesscluster_tools.py index c64797aff7..cd1543996d 100644 --- a/paasta_tools/vitesscluster_tools.py +++ b/paasta_tools/vitesscluster_tools.py @@ -224,12 +224,8 @@ def get_vt_admin_config( return config -class VitessDeploymentConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessClusterConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] - zk_address: str - - -class VitessClusterConfigDict(VitessDeploymentConfigDict, total=False): cells: List[CellConfigDict] vitessDashboard: VitessDashboardConfigDict vtadmin: VtAdminConfigDict @@ -247,10 +243,43 @@ class VitessClusterInstanceConfigDict(KubernetesDeploymentConfigDict, total=Fals images: Dict[str, str] -class VitessDeploymentConfig(KubernetesDeploymentConfig): +class VitessClusterConfig(KubernetesDeploymentConfig): + config_dict: VitessClusterInstanceConfigDict + + config_filename_prefix = "vitesscluster" + + def __init__( + self, + service: str, + cluster: str, + instance: str, + config_dict: VitessClusterConfigDict, + branch_dict: Optional[BranchDictV2], + soa_dir: str = DEFAULT_SOA_DIR, + ) -> None: + super().__init__( + cluster=cluster, # superregion + instance=instance, # host-1 + service=service, # vitess + soa_dir=soa_dir, + config_dict=config_dict, + branch_dict=branch_dict, + ) + def get_namespace(self) -> str: return KUBERNETES_NAMESPACE + def get_images(self) -> Dict[str, str]: + vitess_images = self.config_dict.get( + "images", load_system_paasta_config().get_vitess_images() + ) + return { + "vtctld": vitess_images["vtctld_image"], + "vtadmin": vitess_images["vtadmin_image"], + "vtgate": vitess_images["vtgate_image"], + "vttablet": vitess_images["vttablet_image"], + } + def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: # get all K8s container env vars and format their keys to camel case @@ -314,44 +343,6 @@ def get_region(self) -> str: sys.exit(1) return region - def get_update_strategy(self) -> Dict[str, str]: - return {"type": "Immediate"} - - -class VitessClusterConfig(VitessDeploymentConfig): - config_dict: VitessClusterInstanceConfigDict - - config_filename_prefix = "vitesscluster" - - def __init__( - self, - service: str, - cluster: str, - instance: str, - config_dict: VitessClusterConfigDict, - branch_dict: Optional[BranchDictV2], - soa_dir: str = DEFAULT_SOA_DIR, - ) -> None: - super().__init__( - cluster=cluster, # superregion - instance=instance, # host-1 - service=service, # vitess - soa_dir=soa_dir, - config_dict=config_dict, - branch_dict=branch_dict, - ) - - def get_images(self) -> Dict[str, str]: - vitess_images = self.config_dict.get( - "images", load_system_paasta_config().get_vitess_images() - ) - return { - "vtctld": vitess_images["vtctld_image"], - "vtadmin": vitess_images["vtadmin_image"], - "vtgate": vitess_images["vtgate_image"], - "vttablet": vitess_images["vttablet_image"], - } - def get_global_lock_server(self) -> Dict[str, Dict[str, str]]: zk_address = self.config_dict.get("zk_address") return { @@ -399,6 +390,9 @@ def get_cells(self) -> List[CellConfigDict]: cells = self.config_dict.get("cells") return [get_cell_config(cell) for cell in cells] + def get_update_strategy(self) -> Dict[str, str]: + return {"type": "Immediate"} + def get_vitess_config(self) -> VitessClusterConfigDict: vitess_config = VitessClusterConfigDict( namespace=self.get_namespace(), diff --git a/paasta_tools/vitesskeyspace_tools.py b/paasta_tools/vitesskeyspace_tools.py index 90e3067369..d16161d47b 100644 --- a/paasta_tools/vitesskeyspace_tools.py +++ b/paasta_tools/vitesskeyspace_tools.py @@ -1,4 +1,6 @@ +import json import logging +import sys from typing import Any from typing import Dict from typing import List @@ -8,13 +10,17 @@ from typing import Union import service_configuration_lib +from kubernetes.client import ApiClient +from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict from paasta_tools.kubernetes_tools import limit_size_with_hash from paasta_tools.kubernetes_tools import sanitised_cr_name +from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 from paasta_tools.utils import deep_merge_dictionaries from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_service_instance_config from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import load_v2_deployments_json @@ -23,8 +29,6 @@ from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom from paasta_tools.vitesscluster_tools import RequestsDict from paasta_tools.vitesscluster_tools import ResourceConfigDict -from paasta_tools.vitesscluster_tools import VitessDeploymentConfig -from paasta_tools.vitesscluster_tools import VitessDeploymentConfigDict log = logging.getLogger(__name__) @@ -259,8 +263,9 @@ def get_tablet_pool_config( return config -class VitessKeyspaceConfigDict(VitessDeploymentConfigDict, total=False): +class VitessKeyspaceConfigDict(KubernetesDeploymentConfigDict, total=False): name: str + images: Dict[str, str] databaseName: str durabilityPolicy: str turndownPolicy: str @@ -377,7 +382,7 @@ class VitessKeyspaceInstanceConfigDict(KubernetesDeploymentConfigDict, total=Fal vttablet_resources: ResourceConfigDict -class VitessKeyspaceConfig(VitessDeploymentConfig): +class VitessKeyspaceConfig(KubernetesDeploymentConfig): config_dict: VitessKeyspaceInstanceConfigDict config_filename_prefix = "vitesskeyspace" @@ -400,6 +405,9 @@ def __init__( branch_dict=branch_dict, ) + def get_namespace(self) -> str: + return KUBERNETES_NAMESPACE + def get_images(self) -> Dict[str, str]: vitess_images = self.config_dict.get( "images", load_system_paasta_config().get_vitess_images() @@ -411,6 +419,68 @@ def get_images(self) -> Dict[str, str]: "vttablet": vitess_images["vttablet_image"], } + def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: + # get all K8s container env vars and format their keys to camel case + + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + env = [ + api_client.sanitize_for_serialization(env) + for env in self.get_container_env() + ] + return env + + def get_labels(self) -> Dict[str, str]: + # get default labels from parent class to adhere to paasta contract + docker_url = self.get_docker_url( + system_paasta_config=load_system_paasta_config() + ) + git_sha = get_git_sha_from_dockerurl(docker_url) + labels = self.get_kubernetes_metadata(git_sha=git_sha).labels + if "yelp.com/owner" in labels.keys(): + labels["yelp.com/owner"] = "dre_mysql" + return labels + + def get_annotations(self) -> Mapping[str, Any]: + # get required annotations to be added to the formatted resource before creating or updating custom resource + service_namespace_config = load_service_namespace_config( + service=self.service, namespace=self.get_nerve_namespace() + ) + system_paasta_config = load_system_paasta_config() + has_routable_ip = self.has_routable_ip( + service_namespace_config, system_paasta_config + ) + annotations: Mapping[str, Any] = { + "smartstack_registrations": json.dumps(self.get_registrations()), + "paasta.yelp.com/routable_ip": has_routable_ip, + } + return annotations + + def get_vitess_node_affinity(self) -> dict: + # Workaround from https://github.com/kubernetes-client/python/issues/390 + api_client = ApiClient() + node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) + return node_affinity + + def get_region(self) -> str: + superregion = self.get_cluster() + superregion_to_region_map = ( + load_system_paasta_config().get_superregion_to_region_mapping() + ) + region = None + for superregion_prefix in superregion_to_region_map: + if superregion.startswith(superregion_prefix): + region = superregion.replace( + superregion_prefix, superregion_to_region_map[superregion_prefix] + ) + if region is None: + log.error( + f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" + ) + # Exiting early here since region is needed to fetch secrets from vault + sys.exit(1) + return region + def get_global_lock_server(self) -> Dict[str, str]: zk_address = self.config_dict.get("zk_address") return { @@ -419,6 +489,9 @@ def get_global_lock_server(self) -> Dict[str, str]: "rootPath": TOPO_GLOBAL_ROOT, } + def get_update_strategy(self) -> Dict[str, str]: + return {"type": "Immediate"} + def get_vitess_config(self) -> VitessKeyspaceConfigDict: cells = self.config_dict.get("cells") zk_address = self.config_dict.get("zk_address") diff --git a/tests/test_vitesscluster_tools.py b/tests/test_vitesscluster_tools.py index 7ddd99ec52..ad483f8913 100644 --- a/tests/test_vitesscluster_tools.py +++ b/tests/test_vitesscluster_tools.py @@ -3,9 +3,9 @@ from kubernetes.client import V1ObjectMeta from paasta_tools.utils import SystemPaastaConfig -from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_config -from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_configs -from paasta_tools.vitesscluster_tools import VitessClusterConfig +from paasta_tools.vitesscluster_tools import load_vitess_instance_config +from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs +from paasta_tools.vitesscluster_tools import VitessDeploymentConfig CONFIG_DICT = { @@ -20,6 +20,16 @@ "healthcheck_cmd": "fake_cmd", "healthcheck_grace_period_seconds": 60, "healthcheck_mode": "cmd", + "keyspaces": [ + { + "cluster": "fake_cluster", + "keyspace": "fake_keyspaces", + "vttablet_resources": { + "replicas": 1, + "requests": {"cpu": "100m", "memory": "256Mi"}, + }, + } + ], "monitoring": {}, "node_selectors": {"fake_pool": ["fake_pool_value"]}, "paasta_pool": "fake_pool_value", @@ -30,6 +40,7 @@ "requests": {"cpu": "100m", "memory": "256Mi"}, }, "vtctld_resources": {"replicas": 1, "requests": {"cpu": "100m", "memory": "256Mi"}}, + "vtgate_resources": {"replicas": 1, "requests": {"cpu": "100m", "memory": "256Mi"}}, "zk_address": "fake_zk_address", } @@ -46,7 +57,75 @@ VITESS_CONFIG = { "namespace": "paasta-vitessclusters", - "cells": [{"gateway": {"replicas": 0}, "name": "fake_cell"}], + "cells": [ + { + "gateway": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "fake_pool", + "operator": "In", + "values": ["fake_pool_value"], + } + ] + } + ] + } + } + }, + "annotations": { + "paasta.yelp.com/routable_ip": "false", + "smartstack_registrations": '["fake_service.fake_instance"]', + }, + "extraEnv": [ + { + "name": "VAULT_ADDR", + "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + }, + { + "name": "VAULT_CACERT", + "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + }, + { + "name": "VAULT_ROLEID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vtgate-approle-roleid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", + } + }, + }, + { + "name": "VAULT_SECRETID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vtgate-approle-secretid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", + } + }, + }, + ], + "extraFlags": { + "mysql_auth_server_impl": "vault", + "mysql_auth_vault_addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", + "mysql_auth_vault_tls_ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + "mysql_auth_vault_ttl": "60s", + }, + "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, + "replicas": 1, + "resources": { + "limits": {"cpu": "100m", "memory": "256Mi"}, + "requests": {"cpu": "100m", "memory": "256Mi"}, + }, + }, + "name": "fake_cell", + } + ], "globalLockserver": { "external": { "address": "fake_zk_address", @@ -60,6 +139,404 @@ "vtgate": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3", "vttablet": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3", }, + "keyspaces": [ + { + "durabilityPolicy": "none", + "name": "fake_keyspaces", + "partitionings": [ + { + "equal": { + "parts": 1, + "shardTemplate": { + "databaseInitScriptSecret": { + "key": "/etc/init_db.sql", + "volumeName": "keyspace-fake-init-script", + }, + "tabletPools": [ + { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "fake_pool", + "operator": "In", + "values": [ + "fake_pool_value" + ], + } + ] + } + ] + } + } + }, + "annotations": { + "paasta.yelp.com/routable_ip": "false", + "smartstack_registrations": '["fake_service.fake_instance"]', + }, + "cell": "fake_cell", + "dataVolumeClaimTemplate": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + "storageClassName": "ebs-csi-gp3", + }, + "externalDatastore": { + "credentialsSecret": { + "key": "/etc/credentials.yaml", + "volumeName": "vttablet-fake-credentials", + }, + "database": "fake_keyspaces", + "host": "mysql-fake_cluster-primary.dre-devc", + "port": 3306, + "user": "vt_app", + }, + "extraEnv": [ + { + "name": "VAULT_ADDR", + "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + }, + { + "name": "VAULT_CACERT", + "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + }, + { + "name": "TOPOLOGY_FLAGS", + "value": "--topo_implementation " + "zk2 " + "--topo_global_server_address " + "$fake_zk_address " + "--topo_global_root " + "/vitess-paasta/global", + }, + { + "name": "CELL_TOPOLOGY_SERVERS", + "value": "fake_zk_address", + }, + {"name": "DB", "value": "fake_keyspaces"}, + {"name": "KEYSPACE", "value": "fake_keyspaces"}, + {"name": "WEB_PORT", "value": "15000"}, + {"name": "GRPC_PORT", "value": "15999"}, + {"name": "SHARD", "value": "0"}, + {"name": "EXTERNAL_DB", "value": "1"}, + {"name": "ROLE", "value": "replica"}, + { + "name": "VAULT_ROLEID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vttablet-approle-roleid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", + } + }, + }, + { + "name": "VAULT_SECRETID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vttablet-approle-secretid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", + } + }, + }, + ], + "extraLabels": { + "tablet_type": "fake_keyspaces_migration" + }, + "extraVolumeMounts": [ + { + "mountPath": "/etc/vault/all_cas", + "name": "vault-secrets", + "readOnly": True, + }, + { + "mountPath": "/nail/srv", + "name": "srv-configs", + "readOnly": True, + }, + { + "mountPath": "/nail/etc/srv-configs", + "name": "etc-srv-configs", + "readOnly": True, + }, + { + "mountPath": "etc/credentials.yaml", + "name": "vttablet-fake-credentials", + "readOnly": True, + }, + { + "mountPath": "/etc/init_db.sql", + "name": "keyspace-fake-init-script", + "readOnly": True, + }, + ], + "extraVolumes": [ + { + "hostPath": { + "path": "/nail/etc/vault/all_cas" + }, + "name": "vault-secrets", + }, + { + "name": "srv-configs", + "hostPath": {"path": "/nail/srv"}, + }, + { + "name": "etc-srv-configs", + "hostPath": { + "path": "/nail/etc/srv-configs" + }, + }, + { + "hostPath": {"path": "/dev/null"}, + "name": "vttablet-fake-credentials", + }, + { + "hostPath": {"path": "/dev/null"}, + "name": "keyspace-fake-init-script", + }, + ], + "name": "fake_keyspaces_primary", + "replicas": 1, + "type": "externalmaster", + "vttablet": { + "extraFlags": { + "db-credentials-server": "vault", + "db-credentials-vault-addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", + "db-credentials-vault-tls-ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + "db-credentials-vault-ttl": "60s", + "db_charset": "utf8mb4", + "dba_pool_size": "4", + "disable_active_reparents": "true", + "enable-lag-throttler": "true", + "enforce-tableacl-config": "true", + "grpc_max_message_size": "134217728", + "init_tablet_type": "replica", + "keep_logs": "72h", + "log_err_stacks": "true", + "queryserver-config-schema-reload-time": "1800", + "queryserver-config-strict-table-acl": "true", + "table-acl-config": "/nail/srv/configs/vitess_keyspace_acls/acls_for_fake_keyspaces.json", + "table-acl-config-reload-interval": "60s", + "throttle_check_as_check_self": "true", + "throttle_metrics_query": "select " + "max_replication_delay " + "from " + "max_mysql_replication_delay.read_replication_delay;", + "throttle_metrics_threshold": "3", + "vreplication_heartbeat_update_interval": "60", + "vreplication_tablet_type": "REPLICA", + }, + "resources": { + "limits": { + "cpu": "100m", + "memory": "256Mi", + }, + "requests": { + "cpu": "100m", + "memory": "256Mi", + }, + }, + }, + }, + { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + { + "key": "fake_pool", + "operator": "In", + "values": [ + "fake_pool_value" + ], + } + ] + } + ] + } + } + }, + "annotations": { + "paasta.yelp.com/routable_ip": "false", + "smartstack_registrations": '["fake_service.fake_instance"]', + }, + "cell": "fake_cell", + "dataVolumeClaimTemplate": { + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + "storageClassName": "ebs-csi-gp3", + }, + "externalDatastore": { + "credentialsSecret": { + "key": "/etc/credentials.yaml", + "volumeName": "vttablet-fake-credentials", + }, + "database": "fake_keyspaces", + "host": "mysql-fake_cluster-migration.dre-devc", + "port": 3306, + "user": "vt_app", + }, + "extraEnv": [ + { + "name": "VAULT_ADDR", + "value": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + }, + { + "name": "VAULT_CACERT", + "value": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + }, + { + "name": "TOPOLOGY_FLAGS", + "value": "--topo_implementation " + "zk2 " + "--topo_global_server_address " + "$fake_zk_address " + "--topo_global_root " + "/vitess-paasta/global", + }, + { + "name": "CELL_TOPOLOGY_SERVERS", + "value": "fake_zk_address", + }, + {"name": "DB", "value": "fake_keyspaces"}, + {"name": "KEYSPACE", "value": "fake_keyspaces"}, + {"name": "WEB_PORT", "value": "15000"}, + {"name": "GRPC_PORT", "value": "15999"}, + {"name": "SHARD", "value": "0"}, + {"name": "EXTERNAL_DB", "value": "1"}, + {"name": "ROLE", "value": "replica"}, + { + "name": "VAULT_ROLEID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vttablet-approle-roleid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", + } + }, + }, + { + "name": "VAULT_SECRETID", + "valueFrom": { + "secretKeyRef": { + "key": "vault-vttablet-approle-secretid", + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", + } + }, + }, + ], + "extraLabels": { + "tablet_type": "fake_keyspaces_migration" + }, + "extraVolumeMounts": [ + { + "mountPath": "/etc/vault/all_cas", + "name": "vault-secrets", + "readOnly": True, + }, + { + "mountPath": "/nail/srv", + "name": "srv-configs", + "readOnly": True, + }, + { + "mountPath": "/nail/etc/srv-configs", + "name": "etc-srv-configs", + "readOnly": True, + }, + { + "mountPath": "etc/credentials.yaml", + "name": "vttablet-fake-credentials", + "readOnly": True, + }, + { + "mountPath": "/etc/init_db.sql", + "name": "keyspace-fake-init-script", + "readOnly": True, + }, + ], + "extraVolumes": [ + { + "hostPath": { + "path": "/nail/etc/vault/all_cas" + }, + "name": "vault-secrets", + }, + { + "name": "srv-configs", + "hostPath": {"path": "/nail/srv"}, + }, + { + "name": "etc-srv-configs", + "hostPath": { + "path": "/nail/etc/srv-configs" + }, + }, + { + "hostPath": {"path": "/dev/null"}, + "name": "vttablet-fake-credentials", + }, + { + "hostPath": {"path": "/dev/null"}, + "name": "keyspace-fake-init-script", + }, + ], + "name": "fake_keyspaces_migration", + "replicas": 1, + "type": "externalreplica", + "vttablet": { + "extraFlags": { + "db-credentials-server": "vault", + "db-credentials-vault-addr": "https://vault-dre.mock_region-devc.yelpcorp.com:8200", + "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", + "db-credentials-vault-tls-ca": "/etc/vault/all_cas/acm-privateca-mock_region-devc.crt", + "db-credentials-vault-ttl": "60s", + "db_charset": "utf8mb4", + "dba_pool_size": "4", + "disable_active_reparents": "true", + "enable-lag-throttler": "true", + "enforce-tableacl-config": "true", + "grpc_max_message_size": "134217728", + "init_tablet_type": "replica", + "keep_logs": "72h", + "log_err_stacks": "true", + "queryserver-config-schema-reload-time": "1800", + "queryserver-config-strict-table-acl": "true", + "table-acl-config": "/nail/srv/configs/vitess_keyspace_acls/acls_for_fake_keyspaces.json", + "table-acl-config-reload-interval": "60s", + "throttle_check_as_check_self": "true", + "throttle_metrics_query": "select " + "max_replication_delay " + "from " + "max_mysql_replication_delay.migration_replication_delay;", + "throttle_metrics_threshold": "3", + "vreplication_heartbeat_update_interval": "60", + "vreplication_tablet_type": "REPLICA", + }, + "resources": { + "limits": { + "cpu": "100m", + "memory": "256Mi", + }, + "requests": { + "cpu": "100m", + "memory": "256Mi", + }, + }, + }, + }, + ], + }, + } + } + ], + "turndownPolicy": "Immediate", + } + ], "updateStrategy": {"type": "Immediate"}, "vitessDashboard": { "affinity": { @@ -100,7 +577,7 @@ "disable_active_reparents": "true", "security_policy": "read-only", }, - "extraLabels": {}, + "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, "replicas": 1, "resources": { "limits": {"cpu": "100m", "memory": "256Mi"}, @@ -137,7 +614,7 @@ "cells": ["fake_cell"], "extraEnv": [], "extraFlags": {"grpc-allow-reflection": "true"}, - "extraLabels": {}, + "extraLabels": {"tablet_type": "fake_keyspaces_migration"}, "readOnly": False, "replicas": 1, "webResources": { @@ -149,17 +626,17 @@ @pytest.fixture -def mock_vitess_cluster_config(): +def mock_vitess_deployment_config(): with mock.patch.object( - VitessClusterConfig, "get_region", return_value="mock_region-devc" + VitessDeploymentConfig, "get_region", return_value="mock_region-devc" ), mock.patch.object( - VitessClusterConfig, "get_container_env", return_value=[] + VitessDeploymentConfig, "get_container_env", return_value=[] ), mock.patch.object( - VitessClusterConfig, "get_docker_url", return_value="fake_docker_url" + VitessDeploymentConfig, "get_docker_url", return_value="fake_docker_url" ), mock.patch.object( - VitessClusterConfig, "get_cluster", return_value="fake_superregion" + VitessDeploymentConfig, "get_cluster", return_value="fake_superregion" ), mock.patch.object( - VitessClusterConfig, + VitessDeploymentConfig, "get_kubernetes_metadata", return_value=V1ObjectMeta(labels={}), ): @@ -167,19 +644,19 @@ def mock_vitess_cluster_config(): @mock.patch( - "paasta_tools.vitesscluster_tools.load_vitess_cluster_instance_config", + "paasta_tools.vitesscluster_tools.load_vitess_instance_config", autospec=True, ) @mock.patch( "paasta_tools.vitesscluster_tools.load_system_paasta_config", autospec=True, ) -def test_load_vitess_cluster_instance_configs( +def test_load_vitess_service_instance_configs( mock_load_system_paasta_config, - mock_load_vitess_cluster_instance_config, - mock_vitess_cluster_config, + mock_load_vitess_instance_config, + mock_vitess_deployment_config, ): - mock_load_vitess_cluster_instance_config.return_value = VitessClusterConfig( + mock_load_vitess_instance_config.return_value = VitessDeploymentConfig( service="fake_service", instance="fake_instance", cluster="fake_cluster", @@ -188,13 +665,13 @@ def test_load_vitess_cluster_instance_configs( soa_dir="fake_soa_dir", ) mock_load_system_paasta_config.return_value = MOCK_SYSTEM_PAASTA_CONFIG - vitess_cluster_instance_configs = load_vitess_cluster_instance_configs( + vitess_service_instance_configs = load_vitess_service_instance_configs( service="fake_service", soa_dir="fake_soa_dir", cluster="fake_cluster", instance="fake_instance", ) - assert vitess_cluster_instance_configs == VITESS_CONFIG + assert vitess_service_instance_configs == VITESS_CONFIG @mock.patch("paasta_tools.vitesscluster_tools.load_v2_deployments_json", autospec=True) @@ -203,12 +680,12 @@ def test_load_vitess_cluster_instance_configs( autospec=True, ) @mock.patch( - "paasta_tools.vitesscluster_tools.VitessClusterConfig", + "paasta_tools.vitesscluster_tools.VitessDeploymentConfig", autospec=True, ) -def test_load_vitess_cluster_instance_config( - mock_vitess_cluster_config, - mock_load_cluster_instance_config, +def test_load_vitess_instance_config( + mock_vitess_deployment_config, + mock_load_service_instance_config, mock_load_v2_deployments_json, ): mock_config = { @@ -219,7 +696,7 @@ def test_load_vitess_cluster_instance_config( "smartstack": {}, "dependencies": {}, } - vitess_cluster_config = load_vitess_cluster_instance_config( + vitess_deployment_config = load_vitess_instance_config( service="fake_vitesscluster_service", instance="fake_instance", cluster="fake_cluster", @@ -229,7 +706,7 @@ def test_load_vitess_cluster_instance_config( mock_load_v2_deployments_json.assert_called_with( service="fake_vitesscluster_service", soa_dir="/foo/bar" ) - mock_vitess_cluster_config.assert_called_with( + mock_vitess_deployment_config.assert_called_with( service="fake_vitesscluster_service", instance="fake_instance", cluster="fake_cluster", @@ -238,4 +715,4 @@ def test_load_vitess_cluster_instance_config( soa_dir="/foo/bar", ) - assert vitess_cluster_config == mock_vitess_cluster_config.return_value + assert vitess_deployment_config == mock_vitess_deployment_config.return_value From 6b2cb0893bb958dc4610e1b8c282b2f4020b193e Mon Sep 17 00:00:00 2001 From: Vinay Sagar Gonabavi Date: Wed, 4 Sep 2024 12:45:57 -0700 Subject: [PATCH 3/3] Revert "Add separate fileprefix handlers for vitess crs" This reverts commit e8275f655a053015a00f2b33bfeea94d61e8ef5d. --- paasta_tools/cli/cmds/status.py | 16 +- paasta_tools/cli/utils.py | 20 +- paasta_tools/instance/kubernetes.py | 6 - paasta_tools/setup_kubernetes_cr.py | 10 +- paasta_tools/utils.py | 4 - paasta_tools/vitesscell_tools.py | 360 ---------------- paasta_tools/vitesscluster_tools.py | 449 +++++++++++++++++++- paasta_tools/vitesskeyspace_tools.py | 612 --------------------------- 8 files changed, 439 insertions(+), 1038 deletions(-) delete mode 100644 paasta_tools/vitesscell_tools.py delete mode 100644 paasta_tools/vitesskeyspace_tools.py diff --git a/paasta_tools/cli/cmds/status.py b/paasta_tools/cli/cmds/status.py index b23db83d84..32f1324619 100644 --- a/paasta_tools/cli/cmds/status.py +++ b/paasta_tools/cli/cmds/status.py @@ -95,18 +95,14 @@ from paasta_tools.utils import PaastaColors from paasta_tools.utils import remove_ansi_escape_sequences from paasta_tools.utils import SystemPaastaConfig -from paasta_tools.vitesscell_tools import VitessCellConfig -from paasta_tools.vitesscluster_tools import VitessClusterConfig -from paasta_tools.vitesskeyspace_tools import VitessKeyspaceConfig +from paasta_tools.vitesscluster_tools import VitessDeploymentConfig FLINK_STATUS_MAX_THREAD_POOL_WORKERS = 50 ALLOWED_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [ FlinkDeploymentConfig, FlinkEksDeploymentConfig, CassandraClusterDeploymentConfig, - VitessClusterConfig, - VitessCellConfig, - VitessKeyspaceConfig, + VitessDeploymentConfig, KafkaClusterDeploymentConfig, KubernetesDeploymentConfig, EksDeploymentConfig, @@ -119,9 +115,7 @@ FlinkDeploymentConfig, FlinkEksDeploymentConfig, CassandraClusterDeploymentConfig, - VitessClusterConfig, - VitessCellConfig, - VitessKeyspaceConfig, + VitessDeploymentConfig, KafkaClusterDeploymentConfig, KubernetesDeploymentConfig, EksDeploymentConfig, @@ -143,9 +137,7 @@ EKS_DEPLOYMENT_CONFIGS = [ EksDeploymentConfig, FlinkEksDeploymentConfig, - VitessClusterConfig, - VitessCellConfig, - VitessKeyspaceConfig, + VitessDeploymentConfig, ] FLINK_DEPLOYMENT_CONFIGS = [FlinkDeploymentConfig, FlinkEksDeploymentConfig] diff --git a/paasta_tools/cli/utils.py b/paasta_tools/cli/utils.py index 9861de18d9..979fbff996 100644 --- a/paasta_tools/cli/utils.py +++ b/paasta_tools/cli/utils.py @@ -76,9 +76,7 @@ from paasta_tools.utils import PaastaColors from paasta_tools.utils import SystemPaastaConfig from paasta_tools.utils import validate_service_instance -from paasta_tools.vitesscell_tools import load_vitess_cell_instance_config -from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_config -from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_config +from paasta_tools.vitesscluster_tools import load_vitess_instance_config try: from vault_tools.paasta_secret import get_client as get_vault_client @@ -682,13 +680,7 @@ class LongRunningInstanceTypeHandler(NamedTuple): get_service_instance_list, load_kafkacluster_instance_config ), vitesscluster=InstanceTypeHandler( - get_service_instance_list, load_vitess_cluster_instance_config - ), - vitesscell=InstanceTypeHandler( - get_service_instance_list, load_vitess_cell_instance_config - ), - vitesskeyspace=InstanceTypeHandler( - get_service_instance_list, load_vitess_keyspace_instance_config + get_service_instance_list, load_vitess_instance_config ), nrtsearchservice=InstanceTypeHandler( get_service_instance_list, load_nrtsearchservice_instance_config @@ -721,13 +713,7 @@ class LongRunningInstanceTypeHandler(NamedTuple): get_service_instance_list, load_kafkacluster_instance_config ), vitesscluster=LongRunningInstanceTypeHandler( - get_service_instance_list, load_vitess_cluster_instance_config - ), - vitesscell=LongRunningInstanceTypeHandler( - get_service_instance_list, load_vitess_cell_instance_config - ), - vitesskeyspace=LongRunningInstanceTypeHandler( - get_service_instance_list, load_vitess_keyspace_instance_config + get_service_instance_list, load_vitess_instance_config ), nrtsearchservice=LongRunningInstanceTypeHandler( get_service_instance_list, load_nrtsearchservice_instance_config diff --git a/paasta_tools/instance/kubernetes.py b/paasta_tools/instance/kubernetes.py index 9964fc59b8..0a90db4bd9 100644 --- a/paasta_tools/instance/kubernetes.py +++ b/paasta_tools/instance/kubernetes.py @@ -33,9 +33,7 @@ from paasta_tools import monkrelaycluster_tools from paasta_tools import nrtsearchservice_tools from paasta_tools import smartstack_tools -from paasta_tools import vitesscell_tools from paasta_tools import vitesscluster_tools -from paasta_tools import vitesskeyspace_tools from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS from paasta_tools.instance.hpa_metrics_parser import HPAMetricsDict from paasta_tools.instance.hpa_metrics_parser import HPAMetricsParser @@ -59,8 +57,6 @@ "cassandracluster", "kafkacluster", "vitesscluster", - "vitesscell", - "vitesskeyspace", } INSTANCE_TYPES_K8S = { "cassandracluster", @@ -76,8 +72,6 @@ cassandracluster=cassandracluster_tools.cr_id, kafkacluster=kafkacluster_tools.cr_id, vitesscluster=vitesscluster_tools.cr_id, - vitesscell=vitesscell_tools.cr_id, - vitesskeyspace=vitesskeyspace_tools.cr_id, nrtsearchservice=nrtsearchservice_tools.cr_id, nrtsearchserviceeks=nrtsearchservice_tools.cr_id, monkrelaycluster=monkrelaycluster_tools.cr_id, diff --git a/paasta_tools/setup_kubernetes_cr.py b/paasta_tools/setup_kubernetes_cr.py index 50647e1899..5972b4da72 100644 --- a/paasta_tools/setup_kubernetes_cr.py +++ b/paasta_tools/setup_kubernetes_cr.py @@ -49,19 +49,13 @@ from paasta_tools.utils import get_git_sha_from_dockerurl from paasta_tools.utils import load_all_configs from paasta_tools.utils import load_system_paasta_config -from paasta_tools.vitesscell_tools import load_vitess_cell_instance_configs -from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_configs -from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_configs +from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs log = logging.getLogger(__name__) -INSTANCE_TYPE_TO_CONFIG_LOADER = { - "vitesscluster": load_vitess_cluster_instance_configs, - "vitesscell": load_vitess_cell_instance_configs, - "vitesskeyspace": load_vitess_keyspace_instance_configs, -} +INSTANCE_TYPE_TO_CONFIG_LOADER = {"vitesscluster": load_vitess_service_instance_configs} class StdoutKubeClient: diff --git a/paasta_tools/utils.py b/paasta_tools/utils.py index 12c40dd03b..a5d585c8ca 100644 --- a/paasta_tools/utils.py +++ b/paasta_tools/utils.py @@ -137,8 +137,6 @@ "cassandracluster", "kafkacluster", "vitesscluster", - "vitesscell", - "vitesskeyspace", "monkrelays", "nrtsearchservice", "nrtsearchserviceeks", @@ -158,8 +156,6 @@ "cassandracluster": "paasta-cassandraclusters", "kafkacluster": "paasta-kafkaclusters", "vitesscluster": "paasta-vitessclusters", - "vitesscell": "paasta-vitessclusters", - "vitesskeyspace": "paasta-vitessclusters", "nrtsearchservice": "paasta-nrtsearchservices", "nrtsearchserviceeks": "paasta-nrtsearchservices", } diff --git a/paasta_tools/vitesscell_tools.py b/paasta_tools/vitesscell_tools.py deleted file mode 100644 index 25ba088236..0000000000 --- a/paasta_tools/vitesscell_tools.py +++ /dev/null @@ -1,360 +0,0 @@ -import json -import logging -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Mapping -from typing import Optional -from typing import TypedDict -from typing import Union - -import service_configuration_lib -from kubernetes.client import ApiClient - -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict -from paasta_tools.kubernetes_tools import sanitised_cr_name -from paasta_tools.long_running_service_tools import load_service_namespace_config -from paasta_tools.utils import BranchDictV2 -from paasta_tools.utils import deep_merge_dictionaries -from paasta_tools.utils import DEFAULT_SOA_DIR -from paasta_tools.utils import get_git_sha_from_dockerurl -from paasta_tools.utils import load_service_instance_config -from paasta_tools.utils import load_system_paasta_config -from paasta_tools.utils import load_v2_deployments_json -from paasta_tools.vitesscluster_tools import get_formatted_environment_variables -from paasta_tools.vitesscluster_tools import KVEnvVar -from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom -from paasta_tools.vitesscluster_tools import RequestsDict -from paasta_tools.vitesscluster_tools import ResourceConfigDict - - -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) - - -KUBERNETES_NAMESPACE = "paasta-vitessclusters" - - -# Global variables -TOPO_IMPLEMENTATION = "zk2" -TOPO_GLOBAL_ROOT = "/vitess-paasta/global" - -VTGATE_EXTRA_ENV = { - "VAULT_ROLEID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", - "key": "vault-vtgate-approle-roleid", - } - }, - "VAULT_SECRETID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", - "key": "vault-vtgate-approle-secretid", - } - }, -} - - -class GatewayConfigDict(TypedDict, total=False): - affinity: Dict[str, Any] - extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] - extraFlags: Dict[str, str] - extraLabels: Dict[str, str] - replicas: int - resources: Dict[str, Any] - annotations: Mapping[str, Any] - - -class VitessCellConfigDict(KubernetesDeploymentConfigDict, total=False): - name: str - images: Dict[str, str] - allCells: List[str] - globalLockserver: Dict[str, str] - gateway: GatewayConfigDict - - -def get_cell_config( - cell: str, - images: Dict[str, str], - allCells: List[str], - global_lock_server: Dict[str, str], - region: str, - vtgate_resources: ResourceConfigDict, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], -) -> VitessCellConfigDict: - """ - get vtgate config - """ - replicas = vtgate_resources.get("replicas") - requests = vtgate_resources.get( - "requests", RequestsDict(cpu="100m", memory="256Mi") - ) - environment_overrides: Dict[str, Any] = { - "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", - "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - } - environment_overrides.update(VTGATE_EXTRA_ENV) - updated_vtgate_extra_env = ( - get_formatted_environment_variables(environment_overrides) + env - ) - - config = VitessCellConfigDict( - name=cell, - images=images, - allCells=allCells, - globalLockserver=global_lock_server, - gateway=GatewayConfigDict( - affinity={"nodeAffinity": node_affinity}, - extraEnv=updated_vtgate_extra_env, - extraFlags={ - "mysql_auth_server_impl": "vault", - "mysql_auth_vault_addr": f"https://vault-dre.{region}.yelpcorp.com:8200", - "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", - "mysql_auth_vault_tls_ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "mysql_auth_vault_ttl": "60s", - }, - extraLabels=labels, - replicas=replicas, - resources={ - "requests": requests, - "limits": requests, - }, - annotations=annotations, - ), - ) - return config - - -class VitessCellInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): - cell: str - cells: List[str] - zk_address: str - vtgate_resources: ResourceConfigDict - images: Dict[str, str] - - -class VitessCellConfig(KubernetesDeploymentConfig): - config_dict: VitessCellInstanceConfigDict - - config_filename_prefix = "vitesscell" - - def __init__( - self, - service: str, - cluster: str, - instance: str, - config_dict: VitessCellConfigDict, - branch_dict: Optional[BranchDictV2], - soa_dir: str = DEFAULT_SOA_DIR, - ) -> None: - super().__init__( - cluster=cluster, # superregion - instance=instance, # host-1 - service=service, # vitess - soa_dir=soa_dir, - config_dict=config_dict, - branch_dict=branch_dict, - ) - - def get_namespace(self) -> str: - return KUBERNETES_NAMESPACE - - def get_images(self) -> Dict[str, str]: - vitess_images = self.config_dict.get( - "images", load_system_paasta_config().get_vitess_images() - ) - return { - "vtctld": vitess_images["vtctld_image"], - "vtadmin": vitess_images["vtadmin_image"], - "vtgate": vitess_images["vtgate_image"], - "vttablet": vitess_images["vttablet_image"], - } - - def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: - # get all K8s container env vars and format their keys to camel case - - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - env = [ - api_client.sanitize_for_serialization(env) - for env in self.get_container_env() - ] - return env - - def get_labels(self) -> Dict[str, str]: - # get default labels from parent class to adhere to paasta contract - docker_url = self.get_docker_url( - system_paasta_config=load_system_paasta_config() - ) - git_sha = get_git_sha_from_dockerurl(docker_url) - labels = self.get_kubernetes_metadata(git_sha=git_sha).labels - if "yelp.com/owner" in labels.keys(): - labels["yelp.com/owner"] = "dre_mysql" - return labels - - def get_annotations(self) -> Mapping[str, Any]: - # get required annotations to be added to the formatted resource before creating or updating custom resource - service_namespace_config = load_service_namespace_config( - service=self.service, namespace=self.get_nerve_namespace() - ) - system_paasta_config = load_system_paasta_config() - has_routable_ip = self.has_routable_ip( - service_namespace_config, system_paasta_config - ) - annotations: Mapping[str, Any] = { - "smartstack_registrations": json.dumps(self.get_registrations()), - "paasta.yelp.com/routable_ip": has_routable_ip, - } - - return annotations - - def get_vitess_node_affinity(self) -> dict: - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) - return node_affinity - - def get_region(self) -> str: - superregion = self.get_cluster() - superregion_to_region_map = ( - load_system_paasta_config().get_superregion_to_region_mapping() - ) - region = None - for superregion_prefix in superregion_to_region_map: - if superregion.startswith(superregion_prefix): - region = superregion.replace( - superregion_prefix, superregion_to_region_map[superregion_prefix] - ) - if region is None: - log.error( - f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" - ) - # Exiting early here since region is needed to fetch secrets from vault - sys.exit(1) - return region - - def get_global_lock_server(self) -> Dict[str, str]: - zk_address = self.config_dict.get("zk_address") - return { - "implementation": TOPO_IMPLEMENTATION, - "address": zk_address, - "rootPath": TOPO_GLOBAL_ROOT, - } - - def get_vitess_cell_config(self) -> VitessCellConfigDict: - cell = self.config_dict.get("cell") - all_cells = self.config_dict.get("cells") - images = self.get_images() - global_lock_server = self.get_global_lock_server() - region = self.get_region() - vtgate_resources = self.config_dict.get("vtgate_resources") - - formatted_env = self.get_env_variables() - labels = self.get_labels() - node_affinity = self.get_vitess_node_affinity() - annotations = self.get_annotations() - - return get_cell_config( - cell, - images, - all_cells, - global_lock_server, - region, - vtgate_resources, - formatted_env, - labels, - node_affinity, - annotations, - ) - - def validate( - self, - params: List[str] = [ - "cpus", - "security", - "dependencies_reference", - "deploy_group", - ], - ) -> List[str]: - # Use InstanceConfig to validate shared config keys like cpus and mem - # TODO: add mem back to this list once we fix PAASTA-15582 and - # move to using the same units as flink/marathon etc. - error_msgs = super().validate(params=params) - - if error_msgs: - name = self.get_instance() - return [f"{name}: {msg}" for msg in error_msgs] - else: - return [] - - -def load_vitess_cell_instance_config( - service: str, - instance: str, - cluster: str, - load_deployments: bool = True, - soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessCellConfig: - general_config = service_configuration_lib.read_service_configuration( - service, soa_dir=soa_dir - ) - instance_config = load_service_instance_config( - service, instance, "vitesscell", cluster, soa_dir=soa_dir - ) - general_config = deep_merge_dictionaries( - overrides=instance_config, defaults=general_config - ) - - branch_dict: Optional[BranchDictV2] = None - if load_deployments: - deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) - temp_instance_config = VitessCellConfig( - service=service, - cluster=cluster, - instance=instance, - config_dict=general_config, - branch_dict=None, - soa_dir=soa_dir, - ) - branch = temp_instance_config.get_branch() - deploy_group = temp_instance_config.get_deploy_group() - branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) - - vitess_cell_config = VitessCellConfig( - service=service, - cluster=cluster, - instance=instance, - config_dict=general_config, - branch_dict=branch_dict, - soa_dir=soa_dir, - ) - - return vitess_cell_config - - -def load_vitess_cell_instance_configs( - service: str, - instance: str, - cluster: str, - soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessCellConfigDict: - vitess_cell_instance_configs = load_vitess_cell_instance_config( - service, instance, cluster, soa_dir=soa_dir - ).get_vitess_cell_config() - return vitess_cell_instance_configs - - -# TODO: read this from CRD in service configs -def cr_id(service: str, instance: str) -> Mapping[str, str]: - return dict( - group="planetscale.com", - version="v2", - namespace=KUBERNETES_NAMESPACE, - plural="vitesscells", - name=sanitised_cr_name(service, instance), - ) diff --git a/paasta_tools/vitesscluster_tools.py b/paasta_tools/vitesscluster_tools.py index cd1543996d..35212a3840 100644 --- a/paasta_tools/vitesscluster_tools.py +++ b/paasta_tools/vitesscluster_tools.py @@ -14,6 +14,7 @@ from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict +from paasta_tools.kubernetes_tools import limit_size_with_hash from paasta_tools.kubernetes_tools import sanitised_cr_name from paasta_tools.long_running_service_tools import load_service_namespace_config from paasta_tools.utils import BranchDictV2 @@ -35,6 +36,7 @@ # Global variables TOPO_IMPLEMENTATION = "zk2" TOPO_GLOBAL_ROOT = "/vitess-paasta/global" +SOURCE_DB_HOST = "169.254.255.254" WEB_PORT = "15000" GRPC_PORT = "15999" @@ -45,6 +47,41 @@ "GRPC_PORT": GRPC_PORT, } +VTTABLET_EXTRA_ENV = { + "WEB_PORT": WEB_PORT, + "GRPC_PORT": GRPC_PORT, + "SHARD": "0", + "EXTERNAL_DB": "1", + "ROLE": "replica", + "VAULT_ROLEID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", + "key": "vault-vttablet-approle-roleid", + } + }, + "VAULT_SECRETID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", + "key": "vault-vttablet-approle-secretid", + } + }, +} + +VTGATE_EXTRA_ENV = { + "VAULT_ROLEID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid", + "key": "vault-vtgate-approle-roleid", + } + }, + "VAULT_SECRETID": { + "secretKeyRef": { + "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid", + "key": "vault-vtgate-approle-secretid", + } + }, +} + # Extra Flags VTADMIN_EXTRA_FLAGS = {"grpc-allow-reflection": "true"} @@ -54,6 +91,21 @@ "security_policy": "read-only", } +VTTABLET_EXTRA_FLAGS = { + "log_err_stacks": "true", + "grpc_max_message_size": "134217728", + "init_tablet_type": "replica", + "queryserver-config-schema-reload-time": "1800", + "dba_pool_size": "4", + "vreplication_heartbeat_update_interval": "60", + "vreplication_tablet_type": "REPLICA", + "keep_logs": "72h", + "enable-lag-throttler": "true", + "throttle_check_as_check_self": "true", + "db_charset": "utf8mb4", + "disable_active_reparents": "true", +} + class KVEnvVar(TypedDict, total=False): name: str @@ -78,7 +130,13 @@ class ResourceConfigDict(TypedDict, total=False): class GatewayConfigDict(TypedDict, total=False): + affinity: Dict[str, Any] + extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] + extraFlags: Dict[str, str] + extraLabels: Dict[str, str] replicas: int + resources: Dict[str, Any] + annotations: Mapping[str, Any] class CellConfigDict(TypedDict, total=False): @@ -111,6 +169,44 @@ class VtAdminConfigDict(TypedDict, total=False): annotations: Mapping[str, Any] +class VtTabletDict(TypedDict, total=False): + extraFlags: Dict[str, str] + resources: Dict[str, Any] + + +class TabletPoolDict(TypedDict, total=False): + cell: str + name: str + type: str + affinity: Dict[str, Any] + extraLabels: Dict[str, str] + extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] + extraVolumeMounts: List[Dict[str, Any]] + extraVolumes: List[Dict[str, Any]] + replicas: int + vttablet: VtTabletDict + externalDatastore: Dict[str, Any] + dataVolumeClaimTemplate: Dict[str, Any] + annotations: Mapping[str, Any] + + +class ShardTemplateDict(TypedDict, total=False): + databaseInitScriptSecret: Dict[str, str] + tabletPools: List[TabletPoolDict] + + +class PartitioningValueDict(TypedDict, total=False): + parts: int + shardTemplate: ShardTemplateDict + + +class KeyspaceConfigDict(TypedDict, total=False): + durabilityPolicy: str + turndownPolicy: str + partitionings: List[Dict[str, PartitioningValueDict]] + name: str + + def get_formatted_environment_variables( env_vars: Dict[str, Any] ) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: @@ -134,14 +230,48 @@ def get_formatted_environment_variables( def get_cell_config( cell: str, + region: str, + vtgate_resources: ResourceConfigDict, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], ) -> CellConfigDict: """ get vtgate config """ + replicas = vtgate_resources.get("replicas") + requests = vtgate_resources.get( + "requests", RequestsDict(cpu="100m", memory="256Mi") + ) + environment_overrides: Dict[str, Any] = { + "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", + "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + } + environment_overrides.update(VTGATE_EXTRA_ENV) + updated_vtgate_extra_env = ( + get_formatted_environment_variables(environment_overrides) + env + ) + config = CellConfigDict( name=cell, gateway=GatewayConfigDict( - replicas=0, + affinity={"nodeAffinity": node_affinity}, + extraEnv=updated_vtgate_extra_env, + extraFlags={ + "mysql_auth_server_impl": "vault", + "mysql_auth_vault_addr": f"https://vault-dre.{region}.yelpcorp.com:8200", + "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json", + "mysql_auth_vault_tls_ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "mysql_auth_vault_ttl": "60s", + }, + extraLabels=labels, + replicas=replicas, + resources={ + "requests": requests, + "limits": requests, + }, + annotations=annotations, ), ) return config @@ -224,16 +354,254 @@ def get_vt_admin_config( return config -class VitessClusterConfigDict(KubernetesDeploymentConfigDict, total=False): +def get_tablet_pool_config( + cell: str, + db_name: str, + keyspace: str, + host: str, + zk_address: str, + throttle_query_table: str, + throttle_metrics_threshold: str, + tablet_type: str, + region: str, + vttablet_resources: ResourceConfigDict, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], +) -> TabletPoolDict: + """ + get vttablet config + """ + vttablet_extra_flags = VTTABLET_EXTRA_FLAGS.copy() + flag_overrides = { + "throttle_metrics_query": f"select max_replication_delay from max_mysql_replication_delay.{throttle_query_table};", + "throttle_metrics_threshold": throttle_metrics_threshold, + "enforce-tableacl-config": "true", + "table-acl-config": f"/nail/srv/configs/vitess_keyspace_acls/acls_for_{db_name}.json", + "table-acl-config-reload-interval": "60s", + "queryserver-config-strict-table-acl": "true", + "db-credentials-server": "vault", + "db-credentials-vault-addr": f"https://vault-dre.{region}.yelpcorp.com:8200", + "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", + "db-credentials-vault-tls-ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "db-credentials-vault-ttl": "60s", + } + vttablet_extra_flags.update(flag_overrides) + + environment_overrides: Dict[str, Any] = { + "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", + "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", + "TOPOLOGY_FLAGS": f"--topo_implementation {TOPO_IMPLEMENTATION} --topo_global_server_address ${zk_address} --topo_global_root {TOPO_GLOBAL_ROOT}", + "CELL_TOPOLOGY_SERVERS": zk_address, + "DB": db_name, + "KEYSPACE": keyspace, + } + environment_overrides.update(VTTABLET_EXTRA_ENV) + updated_vttablet_extra_env = ( + get_formatted_environment_variables(environment_overrides) + env + ) + + # Add extra pod label to filter + tablet_type_label = limit_size_with_hash(name=f"{db_name}_{tablet_type}", limit=63) + labels.update({"tablet_type": tablet_type_label}) + + try: + type = load_system_paasta_config().get_vitess_tablet_pool_type_mapping()[ + tablet_type + ] + except KeyError: + log.error( + f"Tablet type {tablet_type} not found in system paasta config vitess_tablet_pool_type_mapping" + ) + type = "externalmaster" + + replicas = vttablet_resources.get("replicas") + requests = vttablet_resources.get( + "requests", RequestsDict(cpu="100m", memory="256Mi") + ) + + config = TabletPoolDict( + cell=cell, + name=f"{db_name}_{tablet_type}", + type=type, + affinity={"nodeAffinity": node_affinity}, + extraLabels=labels, + extraEnv=updated_vttablet_extra_env, + extraVolumeMounts=[ + { + "mountPath": "/etc/vault/all_cas", + "name": "vault-secrets", + "readOnly": True, + }, + { + "mountPath": "/nail/srv", + "name": "srv-configs", + "readOnly": True, + }, + { + "mountPath": "/nail/etc/srv-configs", + "name": "etc-srv-configs", + "readOnly": True, + }, + { + "mountPath": "etc/credentials.yaml", + "name": "vttablet-fake-credentials", + "readOnly": True, + }, + { + "mountPath": "/etc/init_db.sql", + "name": "keyspace-fake-init-script", + "readOnly": True, + }, + ], + extraVolumes=[ + {"name": "vault-secrets", "hostPath": {"path": "/nail/etc/vault/all_cas"}}, + { + "name": "srv-configs", + "hostPath": {"path": "/nail/srv"}, + }, + { + "name": "etc-srv-configs", + "hostPath": {"path": "/nail/etc/srv-configs"}, + }, + {"name": "vttablet-fake-credentials", "hostPath": {"path": "/dev/null"}}, + {"name": "keyspace-fake-init-script", "hostPath": {"path": "/dev/null"}}, + ], + replicas=replicas, + vttablet={ + "extraFlags": vttablet_extra_flags, + "resources": { + "requests": requests, + "limits": requests, + }, + }, + externalDatastore={ + "database": db_name, + "host": host, + "port": 3306, + "user": "vt_app", + "credentialsSecret": { + "key": "/etc/credentials.yaml", + "volumeName": "vttablet-fake-credentials", + }, + }, + dataVolumeClaimTemplate={ + "accessModes": ["ReadWriteOnce"], + "resources": {"requests": {"storage": "10Gi"}}, + "storageClassName": "ebs-csi-gp3", + }, + annotations=annotations, + ) + return config + + +def get_keyspaces_config( + cells: List[str], + keyspaces: List[Dict[str, Any]], + zk_address: str, + region: str, + env: List[Union[KVEnvVar, KVEnvVarValueFrom]], + labels: Dict[str, str], + node_affinity: dict, + annotations: Mapping[str, Any], +) -> List[KeyspaceConfigDict]: + """ + get vitess keyspace config + """ + config = [] + + for keyspace_config in keyspaces: + keyspace = keyspace_config["keyspace"] + db_name = keyspace_config["keyspace"] + cluster = keyspace_config["cluster"] + vttablet_resources = keyspace_config.get("vttablet_resources") + + tablet_pools = [] + + # get vttablets + tablet_types = load_system_paasta_config().get_vitess_tablet_types() + for tablet_type in tablet_types: + ecosystem = region.split("-")[-1] + host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}" + + # We use migration_replication delay for migration tablets and read_replication_delay for everything else + # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds + try: + throttling_configs = ( + load_system_paasta_config().get_vitess_throttling_config() + ) + throttle_query_table = throttling_configs[tablet_type][ + "throttle_query_table" + ] + throttle_metrics_threshold = throttling_configs[tablet_type][ + "throttle_metrics_threshold" + ] + except KeyError: + log.error( + f"Throttling configs for tablet type {tablet_type} not found in system paasta config vitess_throttling_configs" + ) + + if cluster.startswith("refresh") or cluster.startswith("sanitized"): + throttle_metrics_threshold = "30" + else: + throttle_metrics_threshold = "3" + + tablet_pools.extend( + [ + get_tablet_pool_config( + cell, + db_name, + keyspace, + host, + zk_address, + throttle_query_table, + throttle_metrics_threshold, + tablet_type, + region, + vttablet_resources, + env, + labels, + node_affinity, + annotations, + ) + for cell in cells + ] + ) + keyspace_config_value = KeyspaceConfigDict( + name=keyspace, + durabilityPolicy="none", + turndownPolicy="Immediate", + partitionings=[ + { + "equal": PartitioningValueDict( + parts=1, + shardTemplate=ShardTemplateDict( + databaseInitScriptSecret={ + "volumeName": "keyspace-fake-init-script", + "key": "/etc/init_db.sql", + }, + tabletPools=tablet_pools, + ), + ) + } + ], + ) + config.append(keyspace_config_value) + return config + + +class VitessDeploymentConfigDict(KubernetesDeploymentConfigDict, total=False): images: Dict[str, str] cells: List[CellConfigDict] vitessDashboard: VitessDashboardConfigDict vtadmin: VtAdminConfigDict + keyspaces: List[KeyspaceConfigDict] updateStrategy: Dict[str, str] globalLockserver: Dict[str, Dict[str, str]] -class VitessClusterInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): +class VitessInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): cells: List[str] zk_address: str vtctld_resources: ResourceConfigDict @@ -241,10 +609,11 @@ class VitessClusterInstanceConfigDict(KubernetesDeploymentConfigDict, total=Fals vttablet_resources: ResourceConfigDict vtadmin_resources: ResourceConfigDict images: Dict[str, str] + keyspaces: List[Dict[str, Any]] -class VitessClusterConfig(KubernetesDeploymentConfig): - config_dict: VitessClusterInstanceConfigDict +class VitessDeploymentConfig(KubernetesDeploymentConfig): + config_dict: VitessInstanceConfigDict config_filename_prefix = "vitesscluster" @@ -253,7 +622,7 @@ def __init__( service: str, cluster: str, instance: str, - config_dict: VitessClusterConfigDict, + config_dict: VitessDeploymentConfigDict, branch_dict: Optional[BranchDictV2], soa_dir: str = DEFAULT_SOA_DIR, ) -> None: @@ -353,6 +722,29 @@ def get_global_lock_server(self) -> Dict[str, Dict[str, str]]: } } + def get_cells(self) -> List[CellConfigDict]: + cells = self.config_dict.get("cells") + region = self.get_region() + vtgate_resources = self.config_dict.get("vtgate_resources") + + formatted_env = self.get_env_variables() + labels = self.get_labels() + node_affinity = self.get_vitess_node_affinity() + annotations = self.get_annotations() + + return [ + get_cell_config( + cell, + region, + vtgate_resources, + formatted_env, + labels, + node_affinity, + annotations, + ) + for cell in cells + ] + def get_vitess_dashboard(self) -> VitessDashboardConfigDict: cells = self.config_dict.get("cells") zk_address = self.config_dict.get("zk_address") @@ -386,21 +778,40 @@ def get_vtadmin(self) -> VtAdminConfigDict: cells, vtadmin_resources, formatted_env, labels, node_affinity, annotations ) - def get_cells(self) -> List[CellConfigDict]: + def get_keyspaces(self) -> List[KeyspaceConfigDict]: cells = self.config_dict.get("cells") - return [get_cell_config(cell) for cell in cells] + zk_address = self.config_dict.get("zk_address") + region = self.get_region() + keyspaces = self.config_dict.get("keyspaces") + + formatted_env = self.get_env_variables() + labels = self.get_labels() + node_affinity = self.get_vitess_node_affinity() + annotations = self.get_annotations() + + return get_keyspaces_config( + cells, + keyspaces, + zk_address, + region, + formatted_env, + labels, + node_affinity, + annotations, + ) def get_update_strategy(self) -> Dict[str, str]: return {"type": "Immediate"} - def get_vitess_config(self) -> VitessClusterConfigDict: - vitess_config = VitessClusterConfigDict( + def get_vitess_config(self) -> VitessDeploymentConfigDict: + vitess_config = VitessDeploymentConfigDict( namespace=self.get_namespace(), images=self.get_images(), globalLockserver=self.get_global_lock_server(), cells=self.get_cells(), vitessDashboard=self.get_vitess_dashboard(), vtadmin=self.get_vtadmin(), + keyspaces=self.get_keyspaces(), updateStrategy=self.get_update_strategy(), ) return vitess_config @@ -426,13 +837,13 @@ def validate( return [] -def load_vitess_cluster_instance_config( +def load_vitess_instance_config( service: str, instance: str, cluster: str, load_deployments: bool = True, soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessClusterConfig: +) -> VitessDeploymentConfig: general_config = service_configuration_lib.read_service_configuration( service, soa_dir=soa_dir ) @@ -446,7 +857,7 @@ def load_vitess_cluster_instance_config( branch_dict: Optional[BranchDictV2] = None if load_deployments: deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) - temp_instance_config = VitessClusterConfig( + temp_instance_config = VitessDeploymentConfig( service=service, cluster=cluster, instance=instance, @@ -458,7 +869,7 @@ def load_vitess_cluster_instance_config( deploy_group = temp_instance_config.get_deploy_group() branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) - vitess_cluster_config = VitessClusterConfig( + vitess_deployment_config = VitessDeploymentConfig( service=service, cluster=cluster, instance=instance, @@ -467,19 +878,19 @@ def load_vitess_cluster_instance_config( soa_dir=soa_dir, ) - return vitess_cluster_config + return vitess_deployment_config -def load_vitess_cluster_instance_configs( +def load_vitess_service_instance_configs( service: str, instance: str, cluster: str, soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessClusterConfigDict: - vitess_cluster_instance_configs = load_vitess_cluster_instance_config( +) -> VitessDeploymentConfigDict: + vitess_service_instance_configs = load_vitess_instance_config( service, instance, cluster, soa_dir=soa_dir ).get_vitess_config() - return vitess_cluster_instance_configs + return vitess_service_instance_configs # TODO: read this from CRD in service configs diff --git a/paasta_tools/vitesskeyspace_tools.py b/paasta_tools/vitesskeyspace_tools.py deleted file mode 100644 index d16161d47b..0000000000 --- a/paasta_tools/vitesskeyspace_tools.py +++ /dev/null @@ -1,612 +0,0 @@ -import json -import logging -import sys -from typing import Any -from typing import Dict -from typing import List -from typing import Mapping -from typing import Optional -from typing import TypedDict -from typing import Union - -import service_configuration_lib -from kubernetes.client import ApiClient - -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig -from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict -from paasta_tools.kubernetes_tools import limit_size_with_hash -from paasta_tools.kubernetes_tools import sanitised_cr_name -from paasta_tools.long_running_service_tools import load_service_namespace_config -from paasta_tools.utils import BranchDictV2 -from paasta_tools.utils import deep_merge_dictionaries -from paasta_tools.utils import DEFAULT_SOA_DIR -from paasta_tools.utils import get_git_sha_from_dockerurl -from paasta_tools.utils import load_service_instance_config -from paasta_tools.utils import load_system_paasta_config -from paasta_tools.utils import load_v2_deployments_json -from paasta_tools.vitesscluster_tools import get_formatted_environment_variables -from paasta_tools.vitesscluster_tools import KVEnvVar -from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom -from paasta_tools.vitesscluster_tools import RequestsDict -from paasta_tools.vitesscluster_tools import ResourceConfigDict - - -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) - - -KUBERNETES_NAMESPACE = "paasta-vitessclusters" - - -# Global variables -TOPO_IMPLEMENTATION = "zk2" -TOPO_GLOBAL_ROOT = "/vitess-paasta/global" -WEB_PORT = "15000" -GRPC_PORT = "15999" - - -# Environment variables -VTTABLET_EXTRA_ENV = { - "WEB_PORT": WEB_PORT, - "GRPC_PORT": GRPC_PORT, - "SHARD": "0", - "EXTERNAL_DB": "1", - "ROLE": "replica", - "VAULT_ROLEID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-roleid", - "key": "vault-vttablet-approle-roleid", - } - }, - "VAULT_SECRETID": { - "secretKeyRef": { - "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vttablet-approle-secretid", - "key": "vault-vttablet-approle-secretid", - } - }, -} - -# Extra Flags -VTTABLET_EXTRA_FLAGS = { - "log_err_stacks": "true", - "grpc_max_message_size": "134217728", - "init_tablet_type": "replica", - "queryserver-config-schema-reload-time": "1800", - "dba_pool_size": "4", - "vreplication_heartbeat_update_interval": "60", - "vreplication_tablet_type": "REPLICA", - "keep_logs": "72h", - "enable-lag-throttler": "true", - "throttle_check_as_check_self": "true", - "db_charset": "utf8mb4", - "disable_active_reparents": "true", -} - - -class VtTabletDict(TypedDict, total=False): - extraFlags: Dict[str, str] - resources: Dict[str, Any] - - -class TabletPoolDict(TypedDict, total=False): - cell: str - name: str - type: str - affinity: Dict[str, Any] - extraLabels: Dict[str, str] - extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]] - extraVolumeMounts: List[Dict[str, Any]] - extraVolumes: List[Dict[str, Any]] - replicas: int - vttablet: VtTabletDict - externalDatastore: Dict[str, Any] - dataVolumeClaimTemplate: Dict[str, Any] - annotations: Mapping[str, Any] - - -class ShardTemplateDict(TypedDict, total=False): - databaseInitScriptSecret: Dict[str, str] - tabletPools: List[TabletPoolDict] - - -class PartitioningValueDict(TypedDict, total=False): - parts: int - shardTemplate: ShardTemplateDict - - -class KeyspaceConfigDict(TypedDict, total=False): - durabilityPolicy: str - turndownPolicy: str - partitionings: List[Dict[str, PartitioningValueDict]] - name: str - - -def get_tablet_pool_config( - cell: str, - db_name: str, - keyspace: str, - host: str, - zk_address: str, - throttle_query_table: str, - throttle_metrics_threshold: str, - tablet_type: str, - region: str, - vttablet_resources: ResourceConfigDict, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], -) -> TabletPoolDict: - """ - get vttablet config - """ - vttablet_extra_flags = VTTABLET_EXTRA_FLAGS.copy() - flag_overrides = { - "throttle_metrics_query": f"select max_replication_delay from max_mysql_replication_delay.{throttle_query_table};", - "throttle_metrics_threshold": throttle_metrics_threshold, - "enforce-tableacl-config": "true", - "table-acl-config": f"/nail/srv/configs/vitess_keyspace_acls/acls_for_{db_name}.json", - "table-acl-config-reload-interval": "60s", - "queryserver-config-strict-table-acl": "true", - "db-credentials-server": "vault", - "db-credentials-vault-addr": f"https://vault-dre.{region}.yelpcorp.com:8200", - "db-credentials-vault-path": "secrets/vitess/vt-tablet/vttablet_credentials.json", - "db-credentials-vault-tls-ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "db-credentials-vault-ttl": "60s", - } - vttablet_extra_flags.update(flag_overrides) - - environment_overrides: Dict[str, Any] = { - "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200", - "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt", - "TOPOLOGY_FLAGS": f"--topo_implementation {TOPO_IMPLEMENTATION} --topo_global_server_address ${zk_address} --topo_global_root {TOPO_GLOBAL_ROOT}", - "CELL_TOPOLOGY_SERVERS": zk_address, - "DB": db_name, - "KEYSPACE": keyspace, - } - environment_overrides.update(VTTABLET_EXTRA_ENV) - updated_vttablet_extra_env = ( - get_formatted_environment_variables(environment_overrides) + env - ) - - # Add extra pod label to filter - tablet_type_label = limit_size_with_hash(name=f"{db_name}_{tablet_type}", limit=63) - labels.update({"tablet_type": tablet_type_label}) - - try: - type = load_system_paasta_config().get_vitess_tablet_pool_type_mapping()[ - tablet_type - ] - except KeyError: - log.error( - f"Tablet type {tablet_type} not found in system paasta config vitess_tablet_pool_type_mapping" - ) - type = "externalmaster" - - replicas = vttablet_resources.get("replicas") - requests = vttablet_resources.get( - "requests", RequestsDict(cpu="100m", memory="256Mi") - ) - - config = TabletPoolDict( - cell=cell, - name=f"{db_name}_{tablet_type}", - type=type, - affinity={"nodeAffinity": node_affinity}, - extraLabels=labels, - extraEnv=updated_vttablet_extra_env, - extraVolumeMounts=[ - { - "mountPath": "/etc/vault/all_cas", - "name": "vault-secrets", - "readOnly": True, - }, - { - "mountPath": "/nail/srv", - "name": "srv-configs", - "readOnly": True, - }, - { - "mountPath": "/nail/etc/srv-configs", - "name": "etc-srv-configs", - "readOnly": True, - }, - { - "mountPath": "etc/credentials.yaml", - "name": "vttablet-fake-credentials", - "readOnly": True, - }, - { - "mountPath": "/etc/init_db.sql", - "name": "keyspace-fake-init-script", - "readOnly": True, - }, - ], - extraVolumes=[ - {"name": "vault-secrets", "hostPath": {"path": "/nail/etc/vault/all_cas"}}, - { - "name": "srv-configs", - "hostPath": {"path": "/nail/srv"}, - }, - { - "name": "etc-srv-configs", - "hostPath": {"path": "/nail/etc/srv-configs"}, - }, - {"name": "vttablet-fake-credentials", "hostPath": {"path": "/dev/null"}}, - {"name": "keyspace-fake-init-script", "hostPath": {"path": "/dev/null"}}, - ], - replicas=replicas, - vttablet={ - "extraFlags": vttablet_extra_flags, - "resources": { - "requests": requests, - "limits": requests, - }, - }, - externalDatastore={ - "database": db_name, - "host": host, - "port": 3306, - "user": "vt_app", - "credentialsSecret": { - "key": "/etc/credentials.yaml", - "volumeName": "vttablet-fake-credentials", - }, - }, - dataVolumeClaimTemplate={ - "accessModes": ["ReadWriteOnce"], - "resources": {"requests": {"storage": "10Gi"}}, - "storageClassName": "ebs-csi-gp3", - }, - annotations=annotations, - ) - return config - - -class VitessKeyspaceConfigDict(KubernetesDeploymentConfigDict, total=False): - name: str - images: Dict[str, str] - databaseName: str - durabilityPolicy: str - turndownPolicy: str - partitionings: List[Dict[str, PartitioningValueDict]] - updateStrategy: Dict[str, str] - globalLockserver: Dict[str, str] - zoneMap: Dict[str, Any] - - -def get_keyspace_config( - cells: List[str], - keyspace: str, - cluster: str, - vttablet_resources: ResourceConfigDict, - images: Dict[str, str], - update_strategy: Dict[str, str], - global_lockserver: Dict[str, str], - zk_address: str, - region: str, - env: List[Union[KVEnvVar, KVEnvVarValueFrom]], - labels: Dict[str, str], - node_affinity: dict, - annotations: Mapping[str, Any], -) -> VitessKeyspaceConfigDict: - """ - get vitess keyspace config - """ - db_name = keyspace - - tablet_pools = [] - - # get vttablets - tablet_types = load_system_paasta_config().get_vitess_tablet_types() - for tablet_type in tablet_types: - ecosystem = region.split("-")[-1] - host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}" - - # We use migration_replication delay for migration tablets and read_replication_delay for everything else - # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds - try: - throttling_configs = ( - load_system_paasta_config().get_vitess_throttling_config() - ) - throttle_query_table = throttling_configs[tablet_type][ - "throttle_query_table" - ] - throttle_metrics_threshold = throttling_configs[tablet_type][ - "throttle_metrics_threshold" - ] - except KeyError: - log.error( - f"Throttling configs for tablet type {tablet_type} not found in system paasta config vitess_throttling_configs" - ) - - if cluster.startswith("refresh") or cluster.startswith("sanitized"): - throttle_metrics_threshold = "30" - else: - throttle_metrics_threshold = "3" - - tablet_pools.extend( - [ - get_tablet_pool_config( - cell, - db_name, - keyspace, - host, - zk_address, - throttle_query_table, - throttle_metrics_threshold, - tablet_type, - region, - vttablet_resources, - env, - labels, - node_affinity, - annotations, - ) - for cell in cells - ] - ) - vitess_keyspace_config = VitessKeyspaceConfigDict( - name=keyspace, - durabilityPolicy="none", - turndownPolicy="Immediate", - images=images, - databaseName=db_name, - updateStrategy=update_strategy, - globalLockserver=global_lockserver, - partitionings=[ - { - "equal": PartitioningValueDict( - parts=1, - shardTemplate=ShardTemplateDict( - databaseInitScriptSecret={ - "volumeName": "keyspace-fake-init-script", - "key": "/etc/init_db.sql", - }, - tabletPools=tablet_pools, - ), - ) - } - ], - zoneMap={}, - ) - return vitess_keyspace_config - - -class VitessKeyspaceInstanceConfigDict(KubernetesDeploymentConfigDict, total=False): - cells: List[str] - zk_address: str - images: Dict[str, str] - keyspace: str - cluster: str - vttablet_resources: ResourceConfigDict - - -class VitessKeyspaceConfig(KubernetesDeploymentConfig): - config_dict: VitessKeyspaceInstanceConfigDict - - config_filename_prefix = "vitesskeyspace" - - def __init__( - self, - service: str, - cluster: str, - instance: str, - config_dict: VitessKeyspaceConfigDict, - branch_dict: Optional[BranchDictV2], - soa_dir: str = DEFAULT_SOA_DIR, - ) -> None: - super().__init__( - cluster=cluster, # superregion - instance=instance, # host-1 - service=service, # vitess - soa_dir=soa_dir, - config_dict=config_dict, - branch_dict=branch_dict, - ) - - def get_namespace(self) -> str: - return KUBERNETES_NAMESPACE - - def get_images(self) -> Dict[str, str]: - vitess_images = self.config_dict.get( - "images", load_system_paasta_config().get_vitess_images() - ) - return { - "vtctld": vitess_images["vtctld_image"], - "vtadmin": vitess_images["vtadmin_image"], - "vtgate": vitess_images["vtgate_image"], - "vttablet": vitess_images["vttablet_image"], - } - - def get_env_variables(self) -> List[Union[KVEnvVar, KVEnvVarValueFrom]]: - # get all K8s container env vars and format their keys to camel case - - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - env = [ - api_client.sanitize_for_serialization(env) - for env in self.get_container_env() - ] - return env - - def get_labels(self) -> Dict[str, str]: - # get default labels from parent class to adhere to paasta contract - docker_url = self.get_docker_url( - system_paasta_config=load_system_paasta_config() - ) - git_sha = get_git_sha_from_dockerurl(docker_url) - labels = self.get_kubernetes_metadata(git_sha=git_sha).labels - if "yelp.com/owner" in labels.keys(): - labels["yelp.com/owner"] = "dre_mysql" - return labels - - def get_annotations(self) -> Mapping[str, Any]: - # get required annotations to be added to the formatted resource before creating or updating custom resource - service_namespace_config = load_service_namespace_config( - service=self.service, namespace=self.get_nerve_namespace() - ) - system_paasta_config = load_system_paasta_config() - has_routable_ip = self.has_routable_ip( - service_namespace_config, system_paasta_config - ) - annotations: Mapping[str, Any] = { - "smartstack_registrations": json.dumps(self.get_registrations()), - "paasta.yelp.com/routable_ip": has_routable_ip, - } - return annotations - - def get_vitess_node_affinity(self) -> dict: - # Workaround from https://github.com/kubernetes-client/python/issues/390 - api_client = ApiClient() - node_affinity = api_client.sanitize_for_serialization(self.get_node_affinity()) - return node_affinity - - def get_region(self) -> str: - superregion = self.get_cluster() - superregion_to_region_map = ( - load_system_paasta_config().get_superregion_to_region_mapping() - ) - region = None - for superregion_prefix in superregion_to_region_map: - if superregion.startswith(superregion_prefix): - region = superregion.replace( - superregion_prefix, superregion_to_region_map[superregion_prefix] - ) - if region is None: - log.error( - f"Region not found for superregion {superregion}. Check superregion_to_region_mapping in system paasta config" - ) - # Exiting early here since region is needed to fetch secrets from vault - sys.exit(1) - return region - - def get_global_lock_server(self) -> Dict[str, str]: - zk_address = self.config_dict.get("zk_address") - return { - "implementation": TOPO_IMPLEMENTATION, - "address": zk_address, - "rootPath": TOPO_GLOBAL_ROOT, - } - - def get_update_strategy(self) -> Dict[str, str]: - return {"type": "Immediate"} - - def get_vitess_config(self) -> VitessKeyspaceConfigDict: - cells = self.config_dict.get("cells") - zk_address = self.config_dict.get("zk_address") - region = self.get_region() - keyspace = self.config_dict.get("keyspace") - cluster = self.config_dict.get("cluster") - vttablet_resources = self.config_dict.get("vttablet_resources") - global_lockserver = self.get_global_lock_server() - images = self.get_images() - update_strategy = self.get_update_strategy() - - formatted_env = self.get_env_variables() - labels = self.get_labels() - node_affinity = self.get_vitess_node_affinity() - annotations = self.get_annotations() - - return get_keyspace_config( - cells, - keyspace, - cluster, - vttablet_resources, - images, - update_strategy, - global_lockserver, - zk_address, - region, - formatted_env, - labels, - node_affinity, - annotations, - ) - - def validate( - self, - params: List[str] = [ - "cpus", - "security", - "dependencies_reference", - "deploy_group", - ], - ) -> List[str]: - # Use InstanceConfig to validate shared config keys like cpus and mem - # TODO: add mem back to this list once we fix PAASTA-15582 and - # move to using the same units as flink/marathon etc. - error_msgs = super().validate(params=params) - - if error_msgs: - name = self.get_instance() - return [f"{name}: {msg}" for msg in error_msgs] - else: - return [] - - -def load_vitess_keyspace_instance_config( - service: str, - instance: str, - cluster: str, - load_deployments: bool = True, - soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessKeyspaceConfig: - general_config = service_configuration_lib.read_service_configuration( - service, soa_dir=soa_dir - ) - instance_config = load_service_instance_config( - service, instance, "vitesskeyspace", cluster, soa_dir=soa_dir - ) - general_config = deep_merge_dictionaries( - overrides=instance_config, defaults=general_config - ) - - branch_dict: Optional[BranchDictV2] = None - if load_deployments: - deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir) - temp_instance_config = VitessKeyspaceConfig( - service=service, - cluster=cluster, - instance=instance, - config_dict=general_config, - branch_dict=None, - soa_dir=soa_dir, - ) - branch = temp_instance_config.get_branch() - deploy_group = temp_instance_config.get_deploy_group() - branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group) - - vitess_keyspace_config = VitessKeyspaceConfig( - service=service, - cluster=cluster, - instance=instance, - config_dict=general_config, - branch_dict=branch_dict, - soa_dir=soa_dir, - ) - - return vitess_keyspace_config - - -def load_vitess_keyspace_instance_configs( - service: str, - instance: str, - cluster: str, - soa_dir: str = DEFAULT_SOA_DIR, -) -> VitessKeyspaceConfigDict: - vitess_keyspace_instance_configs = load_vitess_keyspace_instance_config( - service, instance, cluster, soa_dir=soa_dir - ).get_vitess_config() - return vitess_keyspace_instance_configs - - -# TODO: read this from CRD in service configs -def cr_id(service: str, instance: str) -> Mapping[str, str]: - return dict( - group="planetscale.com", - version="v2", - namespace=KUBERNETES_NAMESPACE, - plural="vitesskeyspaces", - name=sanitised_cr_name(service, instance), - )