diff --git a/paasta_tools/api/api_docs/oapi.yaml b/paasta_tools/api/api_docs/oapi.yaml index a5b4310684..400f627735 100644 --- a/paasta_tools/api/api_docs/oapi.yaml +++ b/paasta_tools/api/api_docs/oapi.yaml @@ -1630,6 +1630,22 @@ paths: /services/{service}/{instance}/remote_run: post: operationId: remote_run + requestBody: + content: + application/json: + schema: + type: object + properties: + interactive: + type: bool + user: + type: string + image: + type: string + required: + - interactive + - user + required: true parameters: - description: Service name in: path @@ -1643,12 +1659,6 @@ paths: required: true schema: type: string - - description: Username - in: query - name: user - required: true - schema: - type: string responses: "200": content: diff --git a/paasta_tools/api/views/instance.py b/paasta_tools/api/views/instance.py index 274fe782af..4c4e161bb6 100644 --- a/paasta_tools/api/views/instance.py +++ b/paasta_tools/api/views/instance.py @@ -395,9 +395,15 @@ def remote_run(request): service = request.swagger_data.get("service") instance = request.swagger_data.get("instance") user = request.swagger_data.get("user") + interactive = request.swagger_data.get("interactive", True) + recreate = request.swagger_data.get("recreate", True) try: - paasta_remote_run_2.remote_run_start(service, instance, user, settings.cluster) + response = paasta_remote_run_2.remote_run_start( + service, instance, user, settings.cluster, interactive, recreate + ) except Exception: error_message = traceback.format_exc() raise ApiFailure(error_message, 500) + + return response diff --git a/paasta_tools/cli/cmds/remote_run_2.py b/paasta_tools/cli/cmds/remote_run_2.py index dd380ccaf1..5896a5aedb 100644 --- a/paasta_tools/cli/cmds/remote_run_2.py +++ b/paasta_tools/cli/cmds/remote_run_2.py @@ -12,9 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json +import os +import subprocess +import sys +import time + from paasta_tools.api.client import get_paasta_oapi_client +from paasta_tools.cli.cmds.check import makefile_responds_to +from paasta_tools.cli.cmds.cook_image import paasta_cook_image from paasta_tools.cli.utils import get_paasta_oapi_api_clustername from paasta_tools.cli.utils import lazy_choices_completer +from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import get_username from paasta_tools.utils import list_services from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import PaastaColors @@ -56,6 +66,31 @@ def add_subparser( help="Run stuff remotely.", description=("'paasta remote-run' runs stuff remotely "), ) + remote_run_parser.add_argument( + "-b", + "--build", + dest="build", + help="Build the image from current directory", + action="store_true", + ) + remote_run_parser.add_argument( + "-y", + "--yelpsoa-config-root", + dest="yelpsoa_config_root", + help="A directory from which yelpsoa-configs should be read from", + default=DEFAULT_SOA_DIR, + ) + remote_run_parser.add_argument( + "-I", + "--interactive", + help=( + 'Run container in interactive mode. If interactive is set the default command will be "bash" ' + 'unless otherwise set by the "--cmd" flag' + ), + action="store_true", + required=False, + default=False, + ) add_common_args_to_parser(remote_run_parser) remote_run_parser.set_defaults(command=remote_run) @@ -67,9 +102,23 @@ def paasta_remote_run( system_paasta_config: SystemPaastaConfig, verbose: int, is_eks: bool = False, + build: bool = False, ) -> int: + output = [] ret_code = 0 + + # TODO: Build + if build and not makefile_responds_to("cook-image"): + print( + "A local Makefile with a 'cook-image' target is required for --build", + file=sys.stderr, + ) + default_tag = "paasta-remote-run-{}-{}".format(service, get_username()) + os.environ["DOCKER_TAG"] = default_tag + paasta_cook_image(args=None, service=service, soa_dir=soa_dir) + # TODO Actually push the image + client = get_paasta_oapi_client( cluster=get_paasta_oapi_api_clustername(cluster=cluster, is_eks=is_eks), system_paasta_config=system_paasta_config, @@ -79,11 +128,16 @@ def paasta_remote_run( exit(1) try: + # TODO add image argument if build response = client.service.remote_run( - service=service, instance=instance, user="qlo" + service=service, + instance=instance, + user=get_username(), ) - print(response) + print("Reponse was: ", response) + response = json.loads(response) except client.api_error as exc: + print(exc, file=sys.stderr) output.append(PaastaColors.red(exc.reason)) ret_code = exc.status except (client.connection_error, client.timeout_error) as exc: @@ -96,14 +150,31 @@ def paasta_remote_run( output.append(str(e)) ret_code = 1 - print("\n".join(output)) + if ret_code: + print("\n".join(output)) + return ret_code + + pod_name, namespace = response["pod_name"], response["namespace"] + exec_command_tmpl = "kubectl{eks}-{cluster} exec -it -n {namespace} {pod} /bin/bash" + exec_command = exec_command_tmpl.format( + eks="-eks" if is_eks else "", cluster=cluster, namespace=namespace, pod=pod_name + ) + print("Pod launched successfully:", pod_name) + + # TODO figure out how to get this to work + # print('Attaching shell') + # cmd = subprocess.Popen(exec_command.split(' ')) + print("Run the following command to enter your service pod") + print(exec_command) return ret_code def remote_run(args) -> int: """Run stuff, but remotely!""" - system_paasta_config = load_system_paasta_config() - paasta_remote_run( + system_paasta_config = load_system_paasta_config( + "/nail/home/qlo/paasta_config/paasta/" + ) + return paasta_remote_run( args.cluster, args.service, args.instance, system_paasta_config, 1, False ) diff --git a/paasta_tools/paasta_remote_run_2.py b/paasta_tools/paasta_remote_run_2.py index ed60a73aa7..0c60d70d0f 100755 --- a/paasta_tools/paasta_remote_run_2.py +++ b/paasta_tools/paasta_remote_run_2.py @@ -1,5 +1,8 @@ +import json from time import sleep +from kubernetes.client.exceptions import ApiException + from paasta_tools.kubernetes.application.controller_wrappers import ( get_application_wrapper, ) @@ -8,44 +11,91 @@ from paasta_tools.utils import DEFAULT_SOA_DIR -def remote_run_start(service, instance, user, cluster): - kube_client = KubeClient() +def remote_run_start(service, instance, user, cluster, interactive, recreate): + # TODO Overriding the kube client config for now as the api has limited permissions + kube_client = KubeClient(config_file="/etc/kubernetes/admin.conf") + + # TODO hardcoded for now is_eks = False + + # Load the service deployment settings deployment = load_kubernetes_service_config_no_cache( service, instance, cluster, DEFAULT_SOA_DIR ) namespace = deployment.get_namespace() + # Set to interactive mode + if interactive: + deployment.config_dict["cmd"] = "sleep 604800" # One week + + # Create the app with a new name formatted_application = deployment.format_kubernetes_app() formatted_application.metadata.name += f"-remote-run-{user}" - pod_name = formatted_application.metadata.name + deployment_name = formatted_application.metadata.name app_wrapper = get_application_wrapper(formatted_application) app_wrapper.load_local_config(DEFAULT_SOA_DIR, cluster, is_eks) - app_wrapper.create(kube_client) + # Launch pod + try: + app_wrapper.create(kube_client) + except ApiException as e: + if e.status == 409: + # Deployment already running + if recreate: + app_wrapper.deep_delete(kube_client) + wait_until_deployment_gone(kube_client, namespace, deployment_name) + app_wrapper.create(kube_client) + else: + raise + + pod = wait_until_pod_running(kube_client, namespace, deployment_name) + + return json.dumps( + {"status": "success", "pod_name": pod.metadata.name, "namespace": namespace} + ) + + +def wait_until_deployment_gone(kube_client, namespace, deployment_name): + for retry in range(10): + pod = find_pod(kube_client, namespace, deployment_name, 1) + if not pod: + return + sleep(5) + raise Exception("Pod still exists!") + + +def find_pod(kube_client, namespace, deployment_name, retries=5): # Get pod status and name - for retry in range(5): + for retry in range(retries): pod_list = kube_client.core.list_namespaced_pod(namespace) matching_pod = None for pod in pod_list.items: - if pod.metadata.name.startswith(pod_name): + if pod.metadata.name.startswith(deployment_name): matching_pod = pod break if not matching_pod: - sleep(1) + sleep(2) continue + return matching_pod + return None + +def wait_until_pod_running(kube_client, namespace, deployment_name): + for retry in range(5): + pod = find_pod(kube_client, namespace, deployment_name) + if not pod: + raise Exception("No matching pod!") if pod.status.phase == "Running": break - elif pod.status.phase != "Initializing": + elif pod.status.phase not in ("Initializing", "Pending"): raise Exception(f"Pod state is {pod.status.phase}") - - if not matching_pod: - raise Exception("No matching pod") - - return {"Status": "Success!", "pod_name": pod.metadata.name, "namespace": namespace} + return pod -def remote_run_stop(): - pass +# def remote_run_stop(): +# TODO Should this happen here or should the client kill the deployment directly? +# Load the service deployment settings +# deployment = load_kubernetes_service_config_no_cache( +# service, instance, cluster, DEFAULT_SOA_DIR +# ) diff --git a/paasta_tools/paastaapi/api/service_api.py b/paasta_tools/paastaapi/api/service_api.py index 7db7aec9fe..8050e75d2b 100644 --- a/paasta_tools/paastaapi/api/service_api.py +++ b/paasta_tools/paastaapi/api/service_api.py @@ -27,6 +27,7 @@ from paasta_tools.paastaapi.model.flink_config import FlinkConfig from paasta_tools.paastaapi.model.flink_job_details import FlinkJobDetails from paasta_tools.paastaapi.model.flink_jobs import FlinkJobs +from paasta_tools.paastaapi.model.inline_object1 import InlineObject1 from paasta_tools.paastaapi.model.inline_response200 import InlineResponse200 from paasta_tools.paastaapi.model.inline_response2001 import InlineResponse2001 from paasta_tools.paastaapi.model.instance_bounce_status import InstanceBounceStatus @@ -1303,7 +1304,7 @@ def __remote_run( self, service, instance, - user, + inline_object1, **kwargs ): """Launch a remote-run pod # noqa: E501 @@ -1311,13 +1312,13 @@ def __remote_run( This method makes a synchronous HTTP request by default. To make an asynchronous HTTP request, please pass async_req=True - >>> thread = api.remote_run(service, instance, user, async_req=True) + >>> thread = api.remote_run(service, instance, inline_object1, async_req=True) >>> result = thread.get() Args: service (str): Service name instance (str): Instance name - user (str): Username + inline_object1 (InlineObject1): Keyword Args: _return_http_data_only (bool): response data without head status @@ -1368,8 +1369,8 @@ def __remote_run( service kwargs['instance'] = \ instance - kwargs['user'] = \ - user + kwargs['inline_object1'] = \ + inline_object1 return self.call_with_http_info(**kwargs) self.remote_run = Endpoint( @@ -1385,12 +1386,12 @@ def __remote_run( 'all': [ 'service', 'instance', - 'user', + 'inline_object1', ], 'required': [ 'service', 'instance', - 'user', + 'inline_object1', ], 'nullable': [ ], @@ -1409,18 +1410,17 @@ def __remote_run( (str,), 'instance': (str,), - 'user': - (str,), + 'inline_object1': + (InlineObject1,), }, 'attribute_map': { 'service': 'service', 'instance': 'instance', - 'user': 'user', }, 'location_map': { 'service': 'path', 'instance': 'path', - 'user': 'query', + 'inline_object1': 'body', }, 'collection_format_map': { } @@ -1429,7 +1429,9 @@ def __remote_run( 'accept': [ 'application/json' ], - 'content_type': [], + 'content_type': [ + 'application/json' + ] }, api_client=api_client, callable=__remote_run diff --git a/paasta_tools/paastaapi/models/__init__.py b/paasta_tools/paastaapi/models/__init__.py index 0d1278ee23..bda54f7889 100644 --- a/paasta_tools/paastaapi/models/__init__.py +++ b/paasta_tools/paastaapi/models/__init__.py @@ -26,6 +26,7 @@ from paasta_tools.paastaapi.model.float_and_error import FloatAndError from paasta_tools.paastaapi.model.hpa_metric import HPAMetric from paasta_tools.paastaapi.model.inline_object import InlineObject +from paasta_tools.paastaapi.model.inline_object1 import InlineObject1 from paasta_tools.paastaapi.model.inline_response200 import InlineResponse200 from paasta_tools.paastaapi.model.inline_response2001 import InlineResponse2001 from paasta_tools.paastaapi.model.instance_bounce_status import InstanceBounceStatus