Merge pull request #3997 from Yelp/jfong/PAASTA-18479-list-namespaces

PAASTA-18479: Add list-namespaces tool

paasta_tools/cli/cli.py

@@ -110,6 +110,7 @@ def add_subparser(command, subparsers):
     "itest": "itest",
     "list-clusters": "list_clusters",
     "list-deploy-queue": "list_deploy_queue",
+    "list-namespaces": "list_namespaces",
     "list": "list",
     "local-run": "local_run",
     "logs": "logs",

paasta_tools/cli/cmds/autoscale.py

@@ -39,18 +39,20 @@ def add_subparser(subparsers):
     )
 
     autoscale_parser.add_argument(
-        "-s", "--service", help="Service that you want to stop. Like 'example_service'."
+        "-s",
+        "--service",
+        help="Service that you want to autoscale. Like 'example_service'.",
     ).completer = lazy_choices_completer(list_services)
     autoscale_parser.add_argument(
         "-i",
         "--instance",
-        help="Instance of the service that you want to stop. Like 'main' or 'canary'.",
+        help="Instance of the service that you want to autoscale. Like 'main' or 'canary'.",
         required=True,
     ).completer = lazy_choices_completer(list_instances)
     autoscale_parser.add_argument(
         "-c",
         "--cluster",
-        help="The PaaSTA cluster that has the service instance you want to stop. Like 'pnw-prod'.",
+        help="The PaaSTA cluster that has the service instance you want to autoscale. Like 'pnw-prod'.",
         required=True,
     ).completer = lazy_choices_completer(list_clusters)
     autoscale_parser.add_argument(

paasta_tools/cli/cmds/list_namespaces.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paasta_tools.cli.utils import get_instance_configs_for_service
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import validate_service_name
+from paasta_tools.spark_tools import SPARK_EXECUTOR_NAMESPACE
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_clusters
+from paasta_tools.utils import list_services
+
+
+def add_subparser(subparsers) -> None:
+    list_parser = subparsers.add_parser(
+        "list-namespaces",
+        help="Lists all k8s namespaces used by instances of a service",
+    )
+    list_parser.add_argument(
+        "-s",
+        "--service",
+        help="Name of the service which you want to list the namespaces for.",
+        required=True,
+    ).completer = lazy_choices_completer(list_services)
+    # Most services likely don't need to filter by cluster/instance, and can add namespaces from all instances
+    list_parser.add_argument(
+        "-i",
+        "--instance",
+        help="Instance of the service that you want to list namespaces for. Like 'main' or 'canary'.",
+        required=False,
+    )
+    list_parser.add_argument(
+        "-c",
+        "--cluster",
+        help="Clusters that you want to list namespaces for. Like 'pnw-prod' or 'norcal-stagef'.",
+        required=False,
+    ).completer = lazy_choices_completer(list_clusters)
+    list_parser.add_argument(
+        "-y",
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        required=False,
+        help="define a different soa config directory",
+    )
+    list_parser.set_defaults(command=paasta_list_namespaces)
+
+
+def paasta_list_namespaces(args):
+    service = args.service
+    soa_dir = args.soa_dir
+    validate_service_name(service, soa_dir)
+
+    namespaces = set()
+    instance_configs = get_instance_configs_for_service(
+        service=service, soa_dir=soa_dir, instances=args.instance, clusters=args.cluster
+    )
+    for instance in instance_configs:
+        # We skip non-k8s instance types
+        if instance.get_instance_type() in ("paasta-native", "adhoc"):
+            continue
+        namespaces.add(instance.get_namespace())
+        # Tron instances are TronActionConfigs
+        if (
+            instance.get_instance_type() == "tron"
+            and instance.get_executor() == "spark"
+        ):
+            # We also need paasta-spark for spark executors
+            namespaces.add(SPARK_EXECUTOR_NAMESPACE)
+
+    # Print in list format to be used in iam_roles
+    print(list(namespaces))
+    return 0

paasta_tools/long_running_service_tools.py

@@ -149,10 +149,6 @@ def __init__(
     def get_bounce_method(self) -> str:
         raise NotImplementedError
 
-    def get_namespace(self) -> str:
-        """Get namespace from config"""
-        raise NotImplementedError
-
     def get_kubernetes_namespace(self) -> str:
         """
         Only needed on kubernetes LongRunningServiceConfig

tests/cli/test_cmds_list_namespaces.py

@@ -0,0 +1,143 @@
+# tests/cli/cmds/test_list_namespaces.py
+from mock import MagicMock
+from mock import patch
+
+from paasta_tools.cli.cmds.list_namespaces import paasta_list_namespaces
+from paasta_tools.spark_tools import SPARK_EXECUTOR_NAMESPACE
+
+
+def test_list_namespaces_no_instances(capfd):
+    mock_args = MagicMock(
+        service="fake_service",
+        instance=None,
+        cluster=None,
+        soa_dir="/fake/soa/dir",
+    )
+    with patch(
+        "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service",
+        return_value=[],
+        autospec=True,
+    ), patch(
+        "paasta_tools.cli.cmds.list_namespaces.validate_service_name",
+        autospec=True,
+    ):
+        assert paasta_list_namespaces(mock_args) == 0
+        stdout, _ = capfd.readouterr()
+        assert stdout.strip() == "[]"
+
+
+def create_mock_instance_config(instance_type, namespace):
+    """
+    Creates a mock InstanceConfig with specified instance_type and namespace.
+
+    :param instance_type: The type of the instance (e.g., "kubernetes", "paasta-native").
+    :param namespace: The namespace associated with the instance.
+    :return: A mock InstanceConfig object.
+    """
+    mock_instance_config = MagicMock()
+    mock_instance_config.get_instance_type.return_value = instance_type
+    mock_instance_config.get_namespace.return_value = namespace
+    return mock_instance_config
+
+
+def test_list_namespaces_with_instances_dupe_ns(capfd):
+    mock_args = MagicMock(
+        service="fake_service",
+        instance=None,
+        cluster=None,
+        soa_dir="/fake/soa/dir",
+    )
+
+    mock_instance_configs = [
+        create_mock_instance_config("kubernetes", "k8s_namespace"),
+        create_mock_instance_config("kubernetes", "k8s_namespace"),
+    ]
+
+    with patch(
+        "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service",
+        return_value=mock_instance_configs,
+        autospec=True,
+    ), patch(
+        "paasta_tools.cli.cmds.list_namespaces.validate_service_name",
+        autospec=True,
+    ):
+        assert paasta_list_namespaces(mock_args) == 0
+        stdout, _ = capfd.readouterr()
+        assert stdout.strip() == "['k8s_namespace']"
+
+
+def test_list_namespaces_tron(capfd):
+    mock_args = MagicMock(
+        service="fake_service",
+        instance=None,
+        cluster=None,
+        soa_dir="/fake/soa/dir",
+    )
+    mock_tron_instance = create_mock_instance_config("tron", "tron")
+    mock_tron_instance.get_executor.return_value = "paasta"
+
+    with patch(
+        "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service",
+        return_value=[mock_tron_instance],
+        autospec=True,
+    ), patch(
+        "paasta_tools.cli.cmds.list_namespaces.validate_service_name",
+        autospec=True,
+    ):
+        assert paasta_list_namespaces(mock_args) == 0
+        stdout, _ = capfd.readouterr()
+        assert "['tron']"
+
+
+def test_list_namespaces_spark(capfd):
+    mock_args = MagicMock(
+        service="fake_service",
+        instance=None,
+        cluster=None,
+        soa_dir="/fake/soa/dir",
+    )
+    mock_spark_instance = create_mock_instance_config("tron", "tron")
+    mock_spark_instance.get_executor.return_value = "spark"
+
+    mock_instance_configs = [
+        create_mock_instance_config("kubernetes", "k8s_namespace"),
+        mock_spark_instance,
+    ]
+
+    with patch(
+        "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service",
+        return_value=mock_instance_configs,
+        autospec=True,
+    ), patch(
+        "paasta_tools.cli.cmds.list_namespaces.validate_service_name",
+        autospec=True,
+    ):
+        assert paasta_list_namespaces(mock_args) == 0
+        stdout, _ = capfd.readouterr()
+        assert f"'{SPARK_EXECUTOR_NAMESPACE}'" in stdout
+        assert "'tron'" in stdout
+        assert "'k8s_namespace'" in stdout
+
+
+def test_list_namespaces_skips_non_k8s_instances(capfd):
+    mock_args = MagicMock(
+        service="fake_service",
+        instance=None,
+        cluster=None,
+        soa_dir="/fake/soa/dir",
+    )
+
+    mock_k8s_instance_config = create_mock_instance_config("eks", "k8s_namespace")
+    mock_adhoc_instance_config = create_mock_instance_config("adhoc", None)
+
+    with patch(
+        "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service",
+        return_value=[mock_k8s_instance_config, mock_adhoc_instance_config],
+        autospec=True,
+    ), patch(
+        "paasta_tools.cli.cmds.list_namespaces.validate_service_name",
+        autospec=True,
+    ):
+        assert paasta_list_namespaces(mock_args) == 0
+        stdout, _ = capfd.readouterr()
+        assert stdout.strip() == "['k8s_namespace']"