More remote_run stuff

paasta_tools/api/api_docs/oapi.yaml

@@ -1630,6 +1630,22 @@ paths:
   /services/{service}/{instance}/remote_run:
     post:
       operationId: remote_run
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                interactive:
+                  type: bool
+                user:
+                  type: string
+                image:
+                  type: string
+              required:
+               - interactive
+               - user
+        required: true
       parameters:
       - description: Service name
         in: path
@@ -1643,12 +1659,6 @@ paths:
         required: true
         schema:
           type: string
-      - description: Username
-        in: query
-        name: user
-        required: true
-        schema:
-          type: string
       responses:
         "200":
           content:

paasta_tools/api/views/instance.py

@@ -395,9 +395,15 @@ def remote_run(request):
     service = request.swagger_data.get("service")
     instance = request.swagger_data.get("instance")
     user = request.swagger_data.get("user")
+    interactive = request.swagger_data.get("interactive", True)
+    recreate = request.swagger_data.get("recreate", True)
 
     try:
-        paasta_remote_run_2.remote_run_start(service, instance, user, settings.cluster)
+        response = paasta_remote_run_2.remote_run_start(
+            service, instance, user, settings.cluster, interactive, recreate
+        )
     except Exception:
         error_message = traceback.format_exc()
         raise ApiFailure(error_message, 500)
+
+    return response

paasta_tools/cli/cmds/remote_run_2.py

@@ -12,9 +12,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import json
+import os
+import subprocess
+import sys
+import time
+
 from paasta_tools.api.client import get_paasta_oapi_client
+from paasta_tools.cli.cmds.check import makefile_responds_to
+from paasta_tools.cli.cmds.cook_image import paasta_cook_image
 from paasta_tools.cli.utils import get_paasta_oapi_api_clustername
 from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import get_username
 from paasta_tools.utils import list_services
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import PaastaColors
@@ -56,6 +66,31 @@ def add_subparser(
         help="Run stuff remotely.",
         description=("'paasta remote-run' runs stuff remotely "),
     )
+    remote_run_parser.add_argument(
+        "-b",
+        "--build",
+        dest="build",
+        help="Build the image from current directory",
+        action="store_true",
+    )
+    remote_run_parser.add_argument(
+        "-y",
+        "--yelpsoa-config-root",
+        dest="yelpsoa_config_root",
+        help="A directory from which yelpsoa-configs should be read from",
+        default=DEFAULT_SOA_DIR,
+    )
+    remote_run_parser.add_argument(
+        "-I",
+        "--interactive",
+        help=(
+            'Run container in interactive mode. If interactive is set the default command will be "bash" '
+            'unless otherwise set by the "--cmd" flag'
+        ),
+        action="store_true",
+        required=False,
+        default=False,
+    )
     add_common_args_to_parser(remote_run_parser)
     remote_run_parser.set_defaults(command=remote_run)
 
@@ -67,9 +102,23 @@ def paasta_remote_run(
     system_paasta_config: SystemPaastaConfig,
     verbose: int,
     is_eks: bool = False,
+    build: bool = False,
 ) -> int:
+
     output = []
     ret_code = 0
+
+    # TODO: Build
+    if build and not makefile_responds_to("cook-image"):
+        print(
+            "A local Makefile with a 'cook-image' target is required for --build",
+            file=sys.stderr,
+        )
+        default_tag = "paasta-remote-run-{}-{}".format(service, get_username())
+        os.environ["DOCKER_TAG"] = default_tag
+        paasta_cook_image(args=None, service=service, soa_dir=soa_dir)
+        # TODO Actually push the image
+
     client = get_paasta_oapi_client(
         cluster=get_paasta_oapi_api_clustername(cluster=cluster, is_eks=is_eks),
         system_paasta_config=system_paasta_config,
@@ -79,11 +128,16 @@ def paasta_remote_run(
         exit(1)
 
     try:
+        # TODO add image argument if build
         response = client.service.remote_run(
-            service=service, instance=instance, user="qlo"
+            service=service,
+            instance=instance,
+            user=get_username(),
         )
-        print(response)
+        print("Reponse was: ", response)
+        response = json.loads(response)
     except client.api_error as exc:
+        print(exc, file=sys.stderr)
         output.append(PaastaColors.red(exc.reason))
         ret_code = exc.status
     except (client.connection_error, client.timeout_error) as exc:
@@ -96,14 +150,31 @@ def paasta_remote_run(
         output.append(str(e))
         ret_code = 1
 
-    print("\n".join(output))
+    if ret_code:
+        print("\n".join(output))
+        return ret_code
+
+    pod_name, namespace = response["pod_name"], response["namespace"]
+    exec_command_tmpl = "kubectl{eks}-{cluster} exec -it -n {namespace} {pod} /bin/bash"
+    exec_command = exec_command_tmpl.format(
+        eks="-eks" if is_eks else "", cluster=cluster, namespace=namespace, pod=pod_name
+    )
+    print("Pod launched successfully:", pod_name)
+
+    # TODO figure out how to get this to work
+    # print('Attaching shell')
+    # cmd = subprocess.Popen(exec_command.split(' '))
+    print("Run the following command to enter your service pod")
+    print(exec_command)
 
     return ret_code
 
 
 def remote_run(args) -> int:
     """Run stuff, but remotely!"""
-    system_paasta_config = load_system_paasta_config()
-    paasta_remote_run(
+    system_paasta_config = load_system_paasta_config(
+        "/nail/home/qlo/paasta_config/paasta/"
+    )
+    return paasta_remote_run(
         args.cluster, args.service, args.instance, system_paasta_config, 1, False
     )

paasta_tools/paasta_remote_run_2.py

@@ -1,5 +1,8 @@
+import json
 from time import sleep
 
+from kubernetes.client.exceptions import ApiException
+
 from paasta_tools.kubernetes.application.controller_wrappers import (
     get_application_wrapper,
 )
@@ -8,44 +11,91 @@
 from paasta_tools.utils import DEFAULT_SOA_DIR
 
 
-def remote_run_start(service, instance, user, cluster):
-    kube_client = KubeClient()
+def remote_run_start(service, instance, user, cluster, interactive, recreate):
+    # TODO Overriding the kube client config for now as the api has limited permissions
+    kube_client = KubeClient(config_file="/etc/kubernetes/admin.conf")
+
+    # TODO hardcoded for now
     is_eks = False
+
+    # Load the service deployment settings
     deployment = load_kubernetes_service_config_no_cache(
         service, instance, cluster, DEFAULT_SOA_DIR
     )
     namespace = deployment.get_namespace()
 
+    # Set to interactive mode
+    if interactive:
+        deployment.config_dict["cmd"] = "sleep 604800"  # One week
+
+    # Create the app with a new name
     formatted_application = deployment.format_kubernetes_app()
     formatted_application.metadata.name += f"-remote-run-{user}"
-    pod_name = formatted_application.metadata.name
+    deployment_name = formatted_application.metadata.name
     app_wrapper = get_application_wrapper(formatted_application)
     app_wrapper.load_local_config(DEFAULT_SOA_DIR, cluster, is_eks)
-    app_wrapper.create(kube_client)
 
+    # Launch pod
+    try:
+        app_wrapper.create(kube_client)
+    except ApiException as e:
+        if e.status == 409:
+            # Deployment already running
+            if recreate:
+                app_wrapper.deep_delete(kube_client)
+                wait_until_deployment_gone(kube_client, namespace, deployment_name)
+                app_wrapper.create(kube_client)
+            else:
+                raise
+
+    pod = wait_until_pod_running(kube_client, namespace, deployment_name)
+
+    return json.dumps(
+        {"status": "success", "pod_name": pod.metadata.name, "namespace": namespace}
+    )
+
+
+def wait_until_deployment_gone(kube_client, namespace, deployment_name):
+    for retry in range(10):
+        pod = find_pod(kube_client, namespace, deployment_name, 1)
+        if not pod:
+            return
+        sleep(5)
+    raise Exception("Pod still exists!")
+
+
+def find_pod(kube_client, namespace, deployment_name, retries=5):
     # Get pod status and name
-    for retry in range(5):
+    for retry in range(retries):
         pod_list = kube_client.core.list_namespaced_pod(namespace)
         matching_pod = None
         for pod in pod_list.items:
-            if pod.metadata.name.startswith(pod_name):
+            if pod.metadata.name.startswith(deployment_name):
                 matching_pod = pod
                 break
 
         if not matching_pod:
-            sleep(1)
+            sleep(2)
             continue
+        return matching_pod
+    return None
 
+
+def wait_until_pod_running(kube_client, namespace, deployment_name):
+    for retry in range(5):
+        pod = find_pod(kube_client, namespace, deployment_name)
+        if not pod:
+            raise Exception("No matching pod!")
         if pod.status.phase == "Running":
             break
-        elif pod.status.phase != "Initializing":
+        elif pod.status.phase not in ("Initializing", "Pending"):
             raise Exception(f"Pod state is {pod.status.phase}")
-
-    if not matching_pod:
-        raise Exception("No matching pod")
-
-    return {"Status": "Success!", "pod_name": pod.metadata.name, "namespace": namespace}
+    return pod
 
 
-def remote_run_stop():
-    pass
+# def remote_run_stop():
+# TODO Should this happen here or should the client kill the deployment directly?
+# Load the service deployment settings
+# deployment = load_kubernetes_service_config_no_cache(
+#    service, instance, cluster, DEFAULT_SOA_DIR
+# )

paasta_tools/paastaapi/api/service_api.py

@@ -27,6 +27,7 @@
 from paasta_tools.paastaapi.model.flink_config import FlinkConfig
 from paasta_tools.paastaapi.model.flink_job_details import FlinkJobDetails
 from paasta_tools.paastaapi.model.flink_jobs import FlinkJobs
+from paasta_tools.paastaapi.model.inline_object1 import InlineObject1
 from paasta_tools.paastaapi.model.inline_response200 import InlineResponse200
 from paasta_tools.paastaapi.model.inline_response2001 import InlineResponse2001
 from paasta_tools.paastaapi.model.instance_bounce_status import InstanceBounceStatus
@@ -1303,21 +1304,21 @@ def __remote_run(
             self,
             service,
             instance,
-            user,
+            inline_object1,
             **kwargs
         ):
             """Launch a remote-run pod  # noqa: E501
 
             This method makes a synchronous HTTP request by default. To make an
             asynchronous HTTP request, please pass async_req=True
 
-            >>> thread = api.remote_run(service, instance, user, async_req=True)
+            >>> thread = api.remote_run(service, instance, inline_object1, async_req=True)
             >>> result = thread.get()
 
             Args:
                 service (str): Service name
                 instance (str): Instance name
-                user (str): Username
+                inline_object1 (InlineObject1):
 
             Keyword Args:
                 _return_http_data_only (bool): response data without head status
@@ -1368,8 +1369,8 @@ def __remote_run(
                 service
             kwargs['instance'] = \
                 instance
-            kwargs['user'] = \
-                user
+            kwargs['inline_object1'] = \
+                inline_object1
             return self.call_with_http_info(**kwargs)
 
         self.remote_run = Endpoint(
@@ -1385,12 +1386,12 @@ def __remote_run(
                 'all': [
                     'service',
                     'instance',
-                    'user',
+                    'inline_object1',
                 ],
                 'required': [
                     'service',
                     'instance',
-                    'user',
+                    'inline_object1',
                 ],
                 'nullable': [
                 ],
@@ -1409,18 +1410,17 @@ def __remote_run(
                         (str,),
                     'instance':
                         (str,),
-                    'user':
-                        (str,),
+                    'inline_object1':
+                        (InlineObject1,),
                 },
                 'attribute_map': {
                     'service': 'service',
                     'instance': 'instance',
-                    'user': 'user',
                 },
                 'location_map': {
                     'service': 'path',
                     'instance': 'path',
-                    'user': 'query',
+                    'inline_object1': 'body',
                 },
                 'collection_format_map': {
                 }
@@ -1429,7 +1429,9 @@ def __remote_run(
                 'accept': [
                     'application/json'
                 ],
-                'content_type': [],
+                'content_type': [
+                    'application/json'
+                ]
             },
             api_client=api_client,
             callable=__remote_run

paasta_tools/paastaapi/models/__init__.py

@@ -26,6 +26,7 @@
 from paasta_tools.paastaapi.model.float_and_error import FloatAndError
 from paasta_tools.paastaapi.model.hpa_metric import HPAMetric
 from paasta_tools.paastaapi.model.inline_object import InlineObject
+from paasta_tools.paastaapi.model.inline_object1 import InlineObject1
 from paasta_tools.paastaapi.model.inline_response200 import InlineResponse200
 from paasta_tools.paastaapi.model.inline_response2001 import InlineResponse2001
 from paasta_tools.paastaapi.model.instance_bounce_status import InstanceBounceStatus