pty.spawn, stop

paasta_tools/api/api_docs/swagger.json

@@ -947,11 +947,21 @@
                         "type": "string"
                     },
                     {
-                        "in": "query",
+                        "in": "body",
                         "description": "Username",
-                        "name": "user",
+                        "name": "json_body",
                         "required": true,
-                        "type": "string"
+                        "schema": {
+                            "type": "object",
+                            "properties": {
+                                "user": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "user"
+                            ]
+                        }
                     }
                 ]
             }

paasta_tools/api/views/instance.py

@@ -391,7 +391,18 @@ def instance_mesh_status(request):
 
 @view_config(route_name="remote_run.stop", request_method="POST", renderer="json")
 def remote_run_stop(request):
-    pass
+    service = request.swagger_data.get("service")
+    instance = request.swagger_data.get("instance")
+    user = request.swagger_data["json_body"].get("user")
+
+    try:
+        response = paasta_remote_run_2.remote_run_stop(
+            service, instance, user, settings.cluster
+        )
+    except:
+        error_message = traceback.format_exc()
+        raise ApiFailure(error_message, 500)
+    return response
 
 
 @view_config(route_name="remote_run.token", request_method="GET", renderer="json")

paasta_tools/cli/cmds/remote_run_2.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 import json
 import os
-import subprocess
+import pty
 import sys
 import time
 
@@ -66,21 +66,27 @@ def add_subparser(
         help="Run stuff remotely.",
         description=("'paasta remote-run' runs stuff remotely "),
     )
-    remote_run_parser.add_argument(
+    subparsers = remote_run_parser.add_subparsers(dest="remote_run_command")
+    start_parser = subparsers.add_parser(
+        "start",
+        help="Start or connect to a remote-run job",
+        description=("Starts or connects to a remote-run-job"),
+    )
+    start_parser.add_argument(
         "-b",
         "--build",
         dest="build",
         help="Build the image from current directory",
         action="store_true",
     )
-    remote_run_parser.add_argument(
+    start_parser.add_argument(
         "-y",
         "--yelpsoa-config-root",
         dest="yelpsoa_config_root",
         help="A directory from which yelpsoa-configs should be read from",
         default=DEFAULT_SOA_DIR,
     )
-    remote_run_parser.add_argument(
+    start_parser.add_argument(
         "-I",
         "--interactive",
         help=(
@@ -91,20 +97,27 @@ def add_subparser(
         required=False,
         default=False,
     )
-    add_common_args_to_parser(remote_run_parser)
+    stop_parser = subparsers.add_parser(
+        "stop",
+        help="Stop! In the name of Paasta",
+        description="Stop your remote-run job if it exists",
+    )
+    add_common_args_to_parser(start_parser)
+    add_common_args_to_parser(stop_parser)
     remote_run_parser.set_defaults(command=remote_run)
 
 
-def paasta_remote_run(
-    cluster: str,
-    service: str,
-    instance: str,
+def paasta_remote_run_start(
+    args,
     system_paasta_config: SystemPaastaConfig,
-    verbose: int,
-    is_eks: bool = False,
-    build: bool = False,
+    verbose: int = 0,
+    is_eks: bool = True,
 ) -> int:
 
+    cluster = args.cluster
+    service = args.service
+    instance = args.instance
+    build = args.build
     output = []
     ret_code = 0
 
@@ -126,14 +139,14 @@ def paasta_remote_run(
     if not client:
         print("Cannot get a paasta-api client")
         exit(1)
+
+    # TODO add image argument if build
     response = client.remote_run.remote_run_start(
         service,
         instance,
         {"user": get_username(), "interactive": True},
     )
     try:
-        # TODO add image argument if build
-        print("Reponse was: ", response)
         response = json.loads(response)
     except client.api_error as exc:
         print(exc, file=sys.stderr)
@@ -153,8 +166,15 @@ def paasta_remote_run(
         print("\n".join(output))
         return ret_code
 
+    if response["status"] == 409:
+        print(
+            "A remote-run container already exists. Run remote-run stop first if you'd like a new one."
+        )
+        attach = input("Would you like to attach to it? y/n ")
+        if attach == "n":
+            return 0
+    print(response)
     pod_name, namespace = response["pod_name"], response["namespace"]
-    print("Pod launched successfully:", pod_name)
 
     try:
         token = client.remote_run.remote_run_token(
@@ -164,7 +184,6 @@ def paasta_remote_run(
     except:
         raise
 
-    # TODO figure out how to get this to work
     exec_command_tmpl = "kubectl{eks}-{cluster} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
     exec_command = exec_command_tmpl.format(
         eks="-eks" if is_eks else "",
@@ -173,17 +192,46 @@ def paasta_remote_run(
         pod=pod_name,
         token=token,
     )
-    print("Running command", exec_command)
-    # cmd = subprocess.Popen(exec_command.split(' '))
+    cmd = pty.spawn(exec_command.split(" "))
 
     return ret_code
 
 
+def paasta_remote_run_stop(
+    args,
+    system_paasta_config: SystemPaastaConfig,
+    verbose: int = 0,
+    is_eks: bool = True,
+) -> int:
+
+    cluster = args.cluster
+    service = args.service
+    instance = args.instance
+
+    ret_code = 0
+
+    client = get_paasta_oapi_client(
+        cluster=get_paasta_oapi_api_clustername(cluster=cluster, is_eks=is_eks),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        exit(1)
+    response = client.remote_run.remote_run_stop(
+        service,
+        instance,
+        {"user": get_username()},
+    )
+    print(response)
+    return ret_code
+
+
 def remote_run(args) -> int:
     """Run stuff, but remotely!"""
     system_paasta_config = load_system_paasta_config(
         "/nail/home/qlo/paasta_config/paasta/"
     )
-    return paasta_remote_run(
-        args.cluster, args.service, args.instance, system_paasta_config, 1, True
-    )
+    if args.remote_run_command == "start":
+        return paasta_remote_run_start(args, system_paasta_config)
+    elif args.remote_run_command == "stop":
+        return paasta_remote_run_stop(args, system_paasta_config)

paasta_tools/paasta_remote_run_2.py

@@ -64,18 +64,19 @@ def remote_run_start(service, instance, user, cluster, interactive, recreate):
     app_wrapper.load_local_config(DEFAULT_SOA_DIR, cluster, is_eks)
 
     # Launch pod
+    status = 200
     try:
         app_wrapper.create(kube_client)
     except ApiException as e:
         if e.status == 409:
             # Job already running
-            return json.dumps({"status": "409", "reason": "Job already running"})
+            status = 409
         raise
 
     pod = wait_until_pod_running(kube_client, namespace, job_name)
 
     return json.dumps(
-        {"status": "200", "pod_name": pod.metadata.name, "namespace": namespace}
+        {"status": status, "pod_name": pod.metadata.name, "namespace": namespace}
     )
 
 
@@ -117,9 +118,24 @@ def wait_until_pod_running(kube_client, namespace, job_name):
     return pod
 
 
-# def remote_run_stop():
-# TODO Should this happen here or should the client kill the deployment directly?
-# Load the service deployment settings
-# deployment = load_kubernetes_service_config_no_cache(
-#    service, instance, cluster, DEFAULT_SOA_DIR
-# )
+def remote_run_stop(service, instance, user, cluster):
+    # TODO Overriding the kube client config for now as the api has limited permissions
+    kube_client = KubeClient(config_file="/etc/kubernetes/admin.conf")
+    is_eks = True
+    if is_eks:
+        deployment = load_eks_service_config_no_cache(
+            service, instance, cluster, DEFAULT_SOA_DIR
+        )
+    else:
+        deployment = load_kubernetes_service_config_no_cache(
+            service, instance, cluster, DEFAULT_SOA_DIR
+        )
+    namespace = deployment.get_namespace()
+    formatted_job = deployment.format_as_kubernetes_job()
+    job_name = f"remote-run-{user}-{formatted_job.metadata.name}"
+    formatted_job.metadata.name = job_name
+
+    app_wrapper = get_application_wrapper(formatted_job)
+    app_wrapper.load_local_config(DEFAULT_SOA_DIR, cluster, is_eks)
+    app_wrapper.deep_delete(kube_client)
+    return json.dumps({"status": 200, "message": "Job successfully removed"})