Merge branch 'master' of https://github.com/Yelp/paasta into MLCOMPUT…

…E-1203_fix_spark_driver_resources

debian/changelog

@@ -1,3 +1,12 @@
+paasta-tools (1.4.6) xenial; urgency=medium
+
+  * 1.4.6 tagged with 'make release'
+    Commit: Merge pull request #3902 from Yelp/u/gonabavi/DREIMP-
+    10725_add_paasta_status_for_vitesscluster  Add paasta status for
+    vitesscluster and mount /nail/etc/srv-configs to VTTablets
+
+ -- Vinay Sagar Gonabavi <[email protected]>  Thu, 27 Jun 2024 10:40:09 -0700
+
 paasta-tools (1.4.5) xenial; urgency=medium
 
   * 1.4.5 tagged with 'make release'

docs/source/generated/paasta_tools.paastaapi.model.instance_status_vitesscluster.rst

@@ -0,0 +1,7 @@
+paasta\_tools.paastaapi.model.instance\_status\_vitesscluster module
+====================================================================
+
+.. automodule:: paasta_tools.paastaapi.model.instance_status_vitesscluster
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/source/generated/paasta_tools.paastaapi.model.rst

@@ -34,6 +34,7 @@ Submodules
    paasta_tools.paastaapi.model.instance_status_kubernetes_autoscaling_status
    paasta_tools.paastaapi.model.instance_status_kubernetes_v2
    paasta_tools.paastaapi.model.instance_status_tron
+   paasta_tools.paastaapi.model.instance_status_vitesscluster
    paasta_tools.paastaapi.model.instance_tasks
    paasta_tools.paastaapi.model.integer_and_error
    paasta_tools.paastaapi.model.kubernetes_container

paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.5"
+__version__ = "1.4.6"

paasta_tools/api/api_docs/oapi.yaml

@@ -339,6 +339,12 @@ components:
     InstanceSpecCassandraCluster:
       description: Cassandra instance spec
       type: object
+    InstanceMetadataVitessCluster:
+      description: VitessCluster instance metadata
+      type: object
+    InstanceSpecVitessCluster:
+      description: VitessCluster instance spec
+      type: object
     InstanceStatus:
       properties:
         adhoc:
@@ -377,6 +383,14 @@ components:
             status:
               $ref: '#/components/schemas/InstanceStatusCassandraCluster'
           type: object
+        vitesscluster:
+          description: Nullable VitessCluster instance status and metadata
+          properties:
+            metadata:
+              $ref: '#/components/schemas/InstanceMetadataVitessCluster'
+            status:
+              $ref: '#/components/schemas/InstanceStatusVitessCluster'
+          type: object
         kafkacluster:
           description: Nullable KafkaCluster instance status and metadata
           properties:
@@ -412,6 +426,9 @@ components:
     InstanceStatusCassandraCluster:
       description: Cassandra instance status
       type: object
+    InstanceStatusVitessCluster:
+      description: VitessCluster instance status
+      type: object
     InstanceStatusKafkaCluster:
       description: Kafka instance status
       type: object

paasta_tools/api/api_docs/swagger.json

@@ -953,6 +953,18 @@
                         }
                     },
                     "description": "Nullable CassandraCluster instance status"
+                },
+                "vitesscluster": {
+                    "type": "object",
+                    "properties": {
+                        "status": {
+                            "$ref": "#/definitions/InstanceStatusVitessCluster"
+                        },
+                        "metadata": {
+                            "$ref": "#/definitions/InstanceMetadataVitessCluster"
+                        }
+                    },
+                    "description": "Nullable VitessCluster instance status and metadata"
                 }
             }
         },
@@ -1737,6 +1749,14 @@
             "type": "object",
             "description": "Cassandra instance status"
         },
+        "InstanceStatusVitessCluster": {
+            "type": "object",
+            "description": "VitessCluster instance status"
+        },
+        "InstanceMetadataVitessCluster": {
+            "type": "object",
+            "description": "VitessCluster instance metadata"
+        },
         "InstanceTasks": {
             "type": "array",
             "description": "List of tasks associated with instance",

paasta_tools/cli/cmds/status.py

@@ -37,6 +37,7 @@
 from typing import Sequence
 from typing import Tuple
 from typing import Type
+from typing import TypedDict
 from typing import Union
 
 import a_sync
@@ -94,12 +95,14 @@
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import remove_ansi_escape_sequences
 from paasta_tools.utils import SystemPaastaConfig
+from paasta_tools.vitesscluster_tools import VitessDeploymentConfig
 
 FLINK_STATUS_MAX_THREAD_POOL_WORKERS = 50
 ALLOWED_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
+    VitessDeploymentConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -112,6 +115,7 @@
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
+    VitessDeploymentConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -1878,6 +1882,250 @@ def print_kafka_status(
     return 0
 
 
+class EtcdLockServerStatus(TypedDict, total=False):
+    observedGeneration: int
+    available: str
+    clientServiceName: str
+
+
+class LockServerStatus(TypedDict, total=False):
+    etcd: EtcdLockServerStatus
+
+
+class VitessClusterCellStatus(TypedDict, total=False):
+    pendingChanges: str
+    gatewayAvailable: str
+
+
+class VitessClusterKeyspaceStatus(TypedDict, total=False):
+    pendingChanges: str
+    cells: List[str]
+    desiredShards: int
+    shards: int
+    readyShards: int
+    updatedShards: int
+    desiredTablets: int
+    tablets: int
+    readyTablets: int
+    updatedTablets: int
+
+
+class VitessDashboardStatus(TypedDict, total=False):
+    available: str
+    serviceName: str
+
+
+class VTAdminStatus(TypedDict, total=False):
+    available: str
+    serviceName: str
+
+
+class OrphanStatus(TypedDict, total=False):
+    reason: str
+    message: str
+
+
+class VitessClusterStatus(TypedDict, total=False):
+    observedGeneration: int
+    globalLockserver: LockServerStatus
+    gatewayServiceName: str
+    vitessDashboard: VitessDashboardStatus
+    cells: Dict[str, VitessClusterCellStatus]
+    keyspaces: Dict[str, VitessClusterKeyspaceStatus]
+    vtadmin: VTAdminStatus
+    orphanedCells: Dict[str, OrphanStatus]
+    orphanedKeyspaces: Dict[str, OrphanStatus]
+
+
+def print_vitess_status(
+    cluster: str,
+    service: str,
+    instance: str,
+    output: List[str],
+    vitess_status: Mapping[str, Any],
+    verbose: int = 0,
+) -> int:
+    tab = "    "
+    indent = 1
+
+    status: VitessClusterStatus = vitess_status.get("status")
+    if status is None:
+        output.append(
+            PaastaColors.red("indent * tab + Vitess cluster is not available yet")
+        )
+        return 1
+
+    output.append(indent * tab + "Vitess Cluster:")
+    indent += 1
+
+    output.append(
+        indent * tab
+        + "Observed Generation: "
+        + str(status.get("observedGeneration", 0))
+    )
+    output.append(
+        indent * tab + "Gateway Service Name: " + status.get("gatewayServiceName", "")
+    )
+
+    output.append(indent * tab + "Cells:")
+    indent += 1
+    cells: Dict[str, VitessClusterCellStatus] = status.get("cells")
+    if not cells:
+        output.append(
+            indent * tab + "Cells: " + PaastaColors.red("No cell status available")
+        )
+        return 0
+    for cell, cell_status in cells.items():
+        gateway_available: str = cell_status.get("gatewayAvailable")
+        if gateway_available == "True":
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - VTGate: {PaastaColors.green('available')}"
+            )
+        else:
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - VTGate: {PaastaColors.red('unavailable')}"
+            )
+        cell_pending_changes: str = cell_status.get("pendingChanges", None)
+        if cell_pending_changes:
+            output.append(indent * tab + f"  Pending Changes: {cell_pending_changes}")
+    indent -= 1
+
+    output.append(indent * tab + "Vitess Dashboard:")
+    indent += 1
+    vitess_dashboard: VitessDashboardStatus = status.get("vitessDashboard")
+    if not vitess_dashboard:
+        output.append(
+            indent * tab
+            + "Vitess Dashboard: "
+            + PaastaColors.red("No dashboard status available")
+        )
+        return 0
+    vitess_dashboard_available: str = vitess_dashboard.get("available", "")
+    vitess_dashboard_service_name: str = vitess_dashboard.get("serviceName", "")
+    if vitess_dashboard_available == "True":
+        output.append(
+            indent * tab
+            + f"Vitess Dashboard: {vitess_dashboard_service_name} - {PaastaColors.green('available')}"
+        )
+    else:
+        output.append(
+            indent * tab
+            + f"Vitess Dashboard: {vitess_dashboard_service_name} - {PaastaColors.red('unavailable')}"
+        )
+    indent -= 1
+
+    output.append(indent * tab + "VTAdmin:")
+    indent += 1
+    vtadmin: VTAdminStatus = status.get("vtadmin")
+    if not vtadmin:
+        output.append(
+            indent * tab + "VTAdmin: " + PaastaColors.red("No VTAdmin status available")
+        )
+        return 0
+    vtadmin_available: str = vtadmin.get("available", "")
+    vtadmin_service_name: str = vtadmin.get("serviceName", "")
+    if vtadmin_available == "True":
+        output.append(
+            indent * tab
+            + f"VTAdmin: {vtadmin_service_name} - {PaastaColors.green('available')}"
+        )
+    else:
+        output.append(
+            indent * tab
+            + f"VTAdmin: {vtadmin_service_name} - {PaastaColors.red('unavailable')}"
+        )
+    indent -= 1
+
+    output.append(indent * tab + "Keyspaces:")
+    indent += 1
+    keyspaces: Dict[str, VitessClusterKeyspaceStatus] = status.get("keyspaces")
+    if not keyspaces:
+        output.append(
+            indent * tab
+            + "Keyspaces: "
+            + PaastaColors.red("No keyspace status available")
+        )
+        return 0
+    for keyspace, keyspace_status in keyspaces.items():
+        output.append(indent * tab + f"Keyspace: {keyspace}")
+        indent += 1
+        keyspace_pending_changes: str = keyspace_status.get("pendingChanges", None)
+        if keyspace_pending_changes:
+            output.append(
+                indent * tab
+                + f"Keyspace: {keyspace} - Pending Changes: {keyspace_pending_changes}"
+            )
+        keyspace_cells: List[str] = keyspace_status.get("cells", [])
+        output.append(indent * tab + f"  Cells: {', '.join(keyspace_cells)}")
+        desired_shards: int = keyspace_status.get("desiredShards", 0)
+        shards: int = keyspace_status.get("shards", 0)
+        ready_shards: int = keyspace_status.get("readyShards", 0)
+        updated_shards: int = keyspace_status.get("updatedShards", 0)
+        output.append(
+            indent * tab
+            + f"  Shards: {shards} observed, {ready_shards}/{desired_shards} ready, {updated_shards}/{desired_shards} updated"
+        )
+        desired_tablets: int = keyspace_status.get("desiredTablets", 0)
+        tablets: int = keyspace_status.get("tablets", 0)
+        ready_tablets: int = keyspace_status.get("readyTablets", 0)
+        updated_tablets: int = keyspace_status.get("updatedTablets", 0)
+        output.append(
+            indent * tab
+            + f"  Tablets: {tablets} observed, {ready_tablets}/{desired_tablets} ready, {updated_tablets}/{desired_tablets} updated"
+        )
+        indent -= 1
+    indent -= 1
+
+    # This is not needed when not using etcd. We use zk instead
+    global_lockserver: LockServerStatus = status.get("globalLockserver", {})
+    if global_lockserver:
+        output.append(indent * tab + "Global Lockserver:")
+        indent += 1
+        etcd: EtcdLockServerStatus = global_lockserver.get("etcd")
+        if etcd:
+            output.append(indent * tab + "Global Lockserver:")
+            indent += 1
+            observed_generation: int = etcd.get("observedGeneration", 0)
+            available: str = etcd.get("available", "")
+            client_service_name: str = etcd.get("clientServiceName", "")
+            output.append(
+                indent * tab
+                + f"Observed Generation: {observed_generation}, Available: {available}, Client Service Name: {client_service_name}"
+            )
+        indent -= 1
+
+    # Orphaned Cells are not mandatorily seen each time
+    orphaned_cells: Dict[str, OrphanStatus] = status.get("orphanedCells", {})
+    if orphaned_cells:
+        output.append(indent * tab + "Orphaned Cells:")
+        indent += 1
+        for cell, orphan_status in orphaned_cells.items():
+            orphaned_cell_reason: str = orphan_status.get("reason", "")
+            orphaned_cell_message: str = orphan_status.get("message", "")
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - Reason: {orphaned_cell_reason}, Message: {orphaned_cell_message}"
+            )
+        indent -= 1
+
+    # Orphaned Keyspaces are not mandatorily seen each time
+    orphaned_keyspaces: Dict[str, OrphanStatus] = status.get("orphanedKeyspaces", {})
+    if orphaned_keyspaces:
+        output.append(indent * tab + "Orphaned Keyspaces:")
+        indent += 1
+        for keyspace, orphan_status in orphaned_keyspaces.items():
+            orphaned_keyspace_reason: str = orphan_status.get("reason", "")
+            orphaned_keyspace_message: str = orphan_status.get("message", "")
+            output.append(
+                indent * tab
+                + f"Keyspace: {keyspace} - Reason: {orphaned_keyspace_reason}, Message: {orphaned_keyspace_message}"
+            )
+        indent -= 1
+    return 0
+
+
 def report_status_for_cluster(
     service: str,
     cluster: str,
@@ -2273,4 +2521,5 @@ def _use_new_paasta_status(args, system_paasta_config) -> bool:
     flinkeks=print_flinkeks_status,
     kafkacluster=print_kafka_status,
     cassandracluster=print_cassandra_status,
+    vitesscluster=print_vitess_status,
 )