From a978a3bfd6f8cf7c956f59beddd3bdb75d76a926 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Thu, 27 Jun 2024 14:24:29 -0700
Subject: [PATCH 01/29] Adding containerd compatability to oom_logger

---
 .../kill_orphaned_docker_containers.py        | 72 -------------------
 paasta_tools/oom_logger.py                    | 60 +++++++++++++---
 requirements-minimal.txt                      |  2 +
 requirements.txt                              |  2 +
 tests/test_oom_logger.py                      |  9 +++
 5 files changed, 63 insertions(+), 82 deletions(-)
 delete mode 100755 paasta_tools/monitoring/kill_orphaned_docker_containers.py

diff --git a/paasta_tools/monitoring/kill_orphaned_docker_containers.py b/paasta_tools/monitoring/kill_orphaned_docker_containers.py
deleted file mode 100755
index 3a2d06e00c..0000000000
--- a/paasta_tools/monitoring/kill_orphaned_docker_containers.py
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env python
-import argparse
-import sys
-
-import a_sync
-
-from paasta_tools import mesos_tools
-from paasta_tools.utils import get_docker_client
-from paasta_tools.utils import get_running_mesos_docker_containers
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(
-        description=(
-            "Cross references running containers with task ids from the mesos slave"
-            " and optionally kills them."
-        )
-    )
-    parser.add_argument(
-        "-f",
-        "--force",
-        action="store_true",
-        help="Actually kill the containers. (defaults to dry-run)",
-    )
-    args = parser.parse_args()
-    return args
-
-
-@a_sync.to_blocking
-async def main():
-    args = parse_args()
-    docker_client = get_docker_client()
-
-    running_mesos_task_ids = [
-        task["id"]
-        for task in mesos_tools.filter_running_tasks(
-            await mesos_tools.get_running_tasks_from_frameworks("")
-        )
-    ]
-    running_mesos_docker_containers = get_running_mesos_docker_containers()
-
-    orphaned_containers = []
-    for container in running_mesos_docker_containers:
-        mesos_task_id = mesos_tools.get_mesos_id_from_container(
-            container=container, client=docker_client
-        )
-        if mesos_task_id not in running_mesos_task_ids:
-            orphaned_containers.append(
-                (container["Names"][0].strip("/"), mesos_task_id)
-            )
-
-    if orphaned_containers:
-        print(
-            "CRIT: Docker containers are orphaned: {}{}".format(
-                ", ".join(
-                    f"{container_name} ({mesos_task_id})"
-                    for container_name, mesos_task_id in orphaned_containers
-                ),
-                " and will be killed" if args.force else "",
-            )
-        )
-        if args.force:
-            for container_name, mesos_task_id in orphaned_containers:
-                docker_client.kill(container_name)
-        sys.exit(1)
-    else:
-        print("OK: All mesos task IDs accounted for")
-        sys.exit(0)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index ecd4d7c8f2..413539581c 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -33,10 +33,15 @@
   destination(paasta_oom_logger);
 };
 """
+import argparse
+import json
 import re
 import sys
 from collections import namedtuple
 
+import grpc
+from containerd.services.containers.v1 import containers_pb2
+from containerd.services.containers.v1 import containers_pb2_grpc
 from docker.errors import APIError
 
 from paasta_tools.cli.utils import get_instance_config
@@ -76,6 +81,16 @@
 )
 
 
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="paasta_oom_logger")
+    parser.add_argument(
+        "--containerd",
+        action="store_true",
+        help="Use containerd to inspect containers",
+    )
+    return parser.parse_args()
+
+
 def capture_oom_events_from_stdin():
     process_name_regex = re.compile(
         r"^\d+\s[a-zA-Z0-9\-]+\s.*\]\s(.+)\sinvoked\soom-killer:"
@@ -136,11 +151,15 @@ def capture_oom_events_from_stdin():
                 break
 
 
-def get_container_env_as_dict(docker_inspect):
+def get_container_env_as_dict(is_cri_containerd: bool, container_inspect):
     env_vars = {}
-    config = docker_inspect.get("Config")
-    if config is not None:
+    if is_cri_containerd:
+        config = container_inspect.get("process")
+        env = config.get("env", [])
+    else:
+        config = container_inspect.get("Config")
         env = config.get("Env", [])
+    if config is not None:
         for i in env:
             name, _, value = i.partition("=")
             env_vars[name] = value
@@ -209,18 +228,32 @@ def send_sfx_event(service, instance, cluster):
         counter.count()
 
 
+def get_containerd_container(
+    is_cri_containerd: bool, container_id: str
+) -> containers_pb2.Container:
+    with grpc.insecure_channel("unix:///run/containerd/containerd.sock") as channel:
+        containersv1 = containers_pb2_grpc.ContainersStub(channel)
+        if is_cri_containerd:
+            namespace = "k8s.io"
+        else:
+            namespace = "moby"
+        return containersv1.Get(
+            containers_pb2.GetContainerRequest(id=container_id),
+            metadata=(("containerd-namespace", namespace),),
+        ).container
+
+
 def main():
     if clog is None:
         print("CLog logger unavailable, exiting.", file=sys.stderr)
         sys.exit(1)
-
+    args = parse_args()
     clog.config.configure(
         scribe_host="169.254.255.254",
         scribe_port=1463,
         monk_disable=False,
         scribe_disable=False,
     )
-
     cluster = load_system_paasta_config().get_cluster()
     client = get_docker_client()
     for (
@@ -229,11 +262,18 @@ def main():
         container_id,
         process_name,
     ) in capture_oom_events_from_stdin():
-        try:
-            docker_inspect = client.inspect_container(resource_id=container_id)
-        except (APIError):
-            continue
-        env_vars = get_container_env_as_dict(docker_inspect)
+        if args.containerd:
+            # then we're using containerd to inspect containers
+            container_info = get_containerd_container(args.containerd, container_id)
+            container_spec_raw = container_info.spec.value.decode("utf-8")
+            container_inspect = json.loads(container_spec_raw)
+        else:
+            # we're using docker to inspect containers
+            try:
+                container_inspect = client.inspect_container(resource_id=container_id)
+            except (APIError):
+                continue
+        env_vars = get_container_env_as_dict(args.containerd, container_inspect)
         service = env_vars.get("PAASTA_SERVICE", "unknown")
         instance = env_vars.get("PAASTA_INSTANCE", "unknown")
         mesos_container_id = env_vars.get("MESOS_CONTAINER_NAME", "mesos-null")
diff --git a/requirements-minimal.txt b/requirements-minimal.txt
index 830ce66851..0e989ec795 100644
--- a/requirements-minimal.txt
+++ b/requirements-minimal.txt
@@ -8,12 +8,14 @@ botocore
 bravado >= 10.2.0
 certifi
 choice >= 0.1
+containerd
 cookiecutter >= 1.4.0
 croniter
 docker
 dulwich >= 0.17.3
 ephemeral-port-reserve >= 1.0.1
 graphviz
+grpcio
 gunicorn
 humanfriendly
 humanize >= 0.5.1
diff --git a/requirements.txt b/requirements.txt
index 1298782be1..0c0598de3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,6 +18,7 @@ certifi==2017.11.5
 chardet==3.0.4
 choice==0.1
 click==6.6
+containerd==1.5.3
 cookiecutter==1.4.0
 croniter==1.3.4
 decorator==4.1.2
@@ -27,6 +28,7 @@ ephemeral-port-reserve==1.1.0
 future==0.16.0
 google-auth==1.2.0
 graphviz==0.8.2
+grpcio==1.62.2
 gunicorn==19.8.1
 http-parser==0.9.0
 humanfriendly==4.18
diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index 2a3c4489a8..4519e9c2c8 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -21,6 +21,7 @@
 from paasta_tools.oom_logger import log_to_clog
 from paasta_tools.oom_logger import LogLine
 from paasta_tools.oom_logger import main
+from paasta_tools.oom_logger import parse_args
 from paasta_tools.oom_logger import send_sfx_event
 
 
@@ -344,6 +345,11 @@ def test_send_sfx_event(mock_get_instance_config):
         assert mock_meteorite.create_counter.return_value.count.call_count == 1
 
 
+@patch("paasta_tools.oom_logger.argparse", autospec=True)
+def test_parse_args(mock_argparse):
+    assert parse_args() == mock_argparse.ArgumentParser.return_value.parse_args()
+
+
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
 @patch("paasta_tools.oom_logger.clog", autospec=True)
 @patch("paasta_tools.oom_logger.send_sfx_event", autospec=True)
@@ -351,7 +357,9 @@ def test_send_sfx_event(mock_get_instance_config):
 @patch("paasta_tools.oom_logger.log_to_clog", autospec=True)
 @patch("paasta_tools.oom_logger.log_to_paasta", autospec=True)
 @patch("paasta_tools.oom_logger.get_docker_client", autospec=True)
+@patch("paasta_tools.oom_logger.parse_args", autospec=True)
 def test_main(
+    mock_parse_args,
     mock_get_docker_client,
     mock_log_to_paasta,
     mock_log_to_clog,
@@ -365,6 +373,7 @@ def test_main(
 ):
 
     mock_sys_stdin.readline.side_effect = sys_stdin
+    mock_parse_args.return_value.containerd = False
     docker_client = Mock(inspect_container=Mock(return_value=docker_inspect))
     mock_get_docker_client.return_value = docker_client
     mock_load_system_paasta_config.return_value.get_cluster.return_value = (

From 070a761d733b9384cc4b00025b72e6df7db10047 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Thu, 4 Jul 2024 13:17:20 -0700
Subject: [PATCH 02/29] Addressing reviews + adding new regex for containerd

---
 debian/paasta-tools.links                     |  1 -
 ...toring.kill_orphaned_docker_containers.rst |  7 ----
 .../generated/paasta_tools.monitoring.rst     |  1 -
 paasta_tools/oom_logger.py                    | 36 ++++++++++++-------
 setup.py                                      |  1 -
 tests/test_oom_logger.py                      | 18 ++++++++--
 6 files changed, 39 insertions(+), 25 deletions(-)
 delete mode 100644 docs/source/generated/paasta_tools.monitoring.kill_orphaned_docker_containers.rst

diff --git a/debian/paasta-tools.links b/debian/paasta-tools.links
index 9c9720a1ff..367a423316 100644
--- a/debian/paasta-tools.links
+++ b/debian/paasta-tools.links
@@ -13,7 +13,6 @@ opt/venvs/paasta-tools/bin/generate_deployments_for_service.py usr/bin/generate_
 opt/venvs/paasta-tools/bin/generate_services_file.py usr/bin/generate_services_file
 opt/venvs/paasta-tools/bin/generate_services_yaml.py usr/bin/generate_services_yaml
 opt/venvs/paasta-tools/bin/generate_authenticating_services.py usr/bin/generate_authenticating_services
-opt/venvs/paasta-tools/bin/kill_orphaned_docker_containers.py usr/bin/kill_orphaned_docker_containers
 opt/venvs/paasta-tools/bin/kubernetes_remove_evicted_pods.py usr/bin/kubernetes_remove_evicted_pods
 opt/venvs/paasta-tools/bin/paasta-api usr/bin/paasta-api
 opt/venvs/paasta-tools/bin/paasta-fsm usr/bin/paasta-fsm
diff --git a/docs/source/generated/paasta_tools.monitoring.kill_orphaned_docker_containers.rst b/docs/source/generated/paasta_tools.monitoring.kill_orphaned_docker_containers.rst
deleted file mode 100644
index 6bac87c0f6..0000000000
--- a/docs/source/generated/paasta_tools.monitoring.kill_orphaned_docker_containers.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-paasta\_tools.monitoring.kill\_orphaned\_docker\_containers module
-==================================================================
-
-.. automodule:: paasta_tools.monitoring.kill_orphaned_docker_containers
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/generated/paasta_tools.monitoring.rst b/docs/source/generated/paasta_tools.monitoring.rst
index ce4749e5d8..3efa3afd1d 100644
--- a/docs/source/generated/paasta_tools.monitoring.rst
+++ b/docs/source/generated/paasta_tools.monitoring.rst
@@ -7,7 +7,6 @@ Submodules
 .. toctree::
 
    paasta_tools.monitoring.check_k8s_api_performance
-   paasta_tools.monitoring.kill_orphaned_docker_containers
 
 Module contents
 ---------------
diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index 413539581c..ecaef995a3 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -38,6 +38,7 @@
 import re
 import sys
 from collections import namedtuple
+from typing import Dict
 
 import grpc
 from containerd.services.containers.v1 import containers_pb2
@@ -86,7 +87,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--containerd",
         action="store_true",
-        help="Use containerd to inspect containers",
+        help="Use containerd to inspect containers, otherwise use docker",
     )
     return parser.parse_args()
 
@@ -103,17 +104,25 @@ def capture_oom_events_from_stdin():
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
         \s.*Task\sin\s/kubepods/(?:[a-zA-Z]+/)? # start of message; non capturing, optional group for the qos cgroup
-        pod[-\w]+/(\w{12})\w+\s # containerid
+        pod[-\w]+/(\w{12}(?:\w{52})?)\w*\s # Match 'pod' followed by alphanumeric and hyphen characters, then capture 12 characters, optionally followed by 52 characters for container id (12 char for docker and 64 char if we're using containerd), and then zero or more word characters
         killed\sas\sa*  # eom
         """,
         re.VERBOSE,
     )
+    oom_regex_kubernetes_containerd_systemd_cgroup = re.compile(
+        r"""
+        ^(\d+)\s # timestamp
+        ([a-zA-Z0-9\-]+) # hostname
+        \s.*oom-kill:.*task_memcg=/system\.slice/.*nerdctl-(\w{64})\w*\.scope,.*$ # loosely match systemd slice and containerid
+        """,
+        re.VERBOSE,
+    )
     oom_regex_kubernetes_structured = re.compile(
         r"""
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
         \s.*oom-kill:.*task_memcg=/kubepods/(?:[a-zA-Z]+/)? # start of message; non-capturing, optional group for the qos cgroup
-        pod[-\w]+/(\w{12})\w+,.*$ # containerid
+        pod[-\w]+/(\w{12}(?:\w{52})?)\w*,.*$ # containerid
         """,
         re.VERBOSE,
     )
@@ -130,6 +139,7 @@ def capture_oom_events_from_stdin():
         oom_regex_kubernetes,
         oom_regex_kubernetes_structured,
         oom_regex_kubernetes_systemd_cgroup,
+        oom_regex_kubernetes_containerd_systemd_cgroup,
     ]
 
     process_name = ""
@@ -151,7 +161,9 @@ def capture_oom_events_from_stdin():
                 break
 
 
-def get_container_env_as_dict(is_cri_containerd: bool, container_inspect):
+def get_container_env_as_dict(
+    is_cri_containerd: bool, container_inspect: dict
+) -> Dict[str, str]:
     env_vars = {}
     if is_cri_containerd:
         config = container_inspect.get("process")
@@ -228,18 +240,12 @@ def send_sfx_event(service, instance, cluster):
         counter.count()
 
 
-def get_containerd_container(
-    is_cri_containerd: bool, container_id: str
-) -> containers_pb2.Container:
+def get_containerd_container(container_id: str) -> containers_pb2.Container:
     with grpc.insecure_channel("unix:///run/containerd/containerd.sock") as channel:
         containersv1 = containers_pb2_grpc.ContainersStub(channel)
-        if is_cri_containerd:
-            namespace = "k8s.io"
-        else:
-            namespace = "moby"
         return containersv1.Get(
             containers_pb2.GetContainerRequest(id=container_id),
-            metadata=(("containerd-namespace", namespace),),
+            metadata=(("containerd-namespace", "k8s.io"),),
         ).container
 
 
@@ -264,7 +270,11 @@ def main():
     ) in capture_oom_events_from_stdin():
         if args.containerd:
             # then we're using containerd to inspect containers
-            container_info = get_containerd_container(args.containerd, container_id)
+            try:
+                container_info = get_containerd_container(container_id)
+            except grpc.RpcError as e:
+                print("An error occurred while getting the container:", e)
+                continue
             container_spec_raw = container_info.spec.value.decode("utf-8")
             container_inspect = json.loads(container_spec_raw)
         else:
diff --git a/setup.py b/setup.py
index ad397bba21..a11ae18f56 100644
--- a/setup.py
+++ b/setup.py
@@ -62,7 +62,6 @@ def get_install_requires():
         "paasta_tools/kubernetes/bin/paasta_cleanup_stale_nodes.py",
         "paasta_tools/kubernetes/bin/paasta_secrets_sync.py",
         "paasta_tools/log_task_lifecycle_events.py",
-        "paasta_tools/monitoring/kill_orphaned_docker_containers.py",
         "paasta_tools/paasta_deploy_tron_jobs",
         "paasta_tools/paasta_execute_docker_command.py",
         "paasta_tools/paasta_remote_run.py",
diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index 4519e9c2c8..1fbfa3d5a5 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -220,7 +220,14 @@ def test_capture_oom_events_from_stdin_kubernetes_qos(
         test_output = []
         for a_tuple in capture_oom_events_from_stdin():
             test_output.append(a_tuple)
-        assert test_output == [(1500316300, "dev37-devc", "0e4a814eda03", "apache2")]
+        assert test_output == [
+            (
+                1500316300,
+                "dev37-devc",
+                "0e4a814eda03622476ff47871e6c397e5b8747af209b44f3b3e1c5289b0f9772",
+                "apache2",
+            )
+        ]
 
 
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
@@ -232,7 +239,14 @@ def test_capture_oom_events_from_stdin_kubernetes_structured_qos(
     test_output = []
     for a_tuple in capture_oom_events_from_stdin():
         test_output.append(a_tuple)
-    assert test_output == [(1500316300, "dev37-devc", "0e4a814eda03", "apache2")]
+    assert test_output == [
+        (
+            1500316300,
+            "dev37-devc",
+            "0e4a814eda030cdc2bb6944078eaefd3278f8f9b3a9725c4ddffb722752a2279",
+            "apache2",
+        )
+    ]
 
 
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)

From ba53ddbf50ab34381a6d7fbbf1feeab3e07e2662 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Thu, 4 Jul 2024 14:31:45 -0700
Subject: [PATCH 03/29] Adding in addition to the nerdctl regex a regex for
 capturing containerd-cri oom

---
 paasta_tools/oom_logger.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index ecaef995a3..1970e60fcf 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -110,6 +110,15 @@ def capture_oom_events_from_stdin():
         re.VERBOSE,
     )
     oom_regex_kubernetes_containerd_systemd_cgroup = re.compile(
+        r"""
+        ^(\d+)\s # timestamp
+        ([a-zA-Z0-9\-]+) # hostname
+        \s.*oom-kill:.*task_memcg=/kubepods\.slice/.* # loosely match systemd slice and containerid
+        cri-containerd-(\w{64}).*$ # containerid
+        """,
+        re.VERBOSE,
+    )
+    oom_regex_kubernetes_nerdctl_systemd_cgroup = re.compile(
         r"""
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
@@ -140,6 +149,7 @@ def capture_oom_events_from_stdin():
         oom_regex_kubernetes_structured,
         oom_regex_kubernetes_systemd_cgroup,
         oom_regex_kubernetes_containerd_systemd_cgroup,
+        oom_regex_kubernetes_nerdctl_systemd_cgroup,
     ]
 
     process_name = ""

From 52b62c8b3b85ec008cc15832dbd31a57a0ee947f Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Tue, 9 Jul 2024 08:29:38 -0700
Subject: [PATCH 04/29] Updating packages for the k8s_itest to pass

---
 requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 0c0598de3a..a974404cf7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,12 +23,12 @@ cookiecutter==1.4.0
 croniter==1.3.4
 decorator==4.1.2
 docker==5.0.3
-dulwich==0.17.3
+dulwich==0.18.4
 ephemeral-port-reserve==1.1.0
 future==0.16.0
 google-auth==1.2.0
 graphviz==0.8.2
-grpcio==1.62.2
+grpcio==1.46.3
 gunicorn==19.8.1
 http-parser==0.9.0
 humanfriendly==4.18
@@ -61,7 +61,7 @@ ply==3.4
 poyo==0.4.0
 progressbar2==3.10.1
 prometheus-client==0.7.1
-protobuf==3.15.0
+protobuf==3.20.3
 py==1.5.2
 pyasn1==0.4.2
 pyasn1-modules==0.2.1
@@ -79,7 +79,7 @@ python-iptables==1.0.1
 python-utils==2.0.1
 pytimeparse==1.1.5
 pytz==2016.10
-pyyaml==6.0.1
+pyyaml==6.0
 repoze.lru==0.6
 requests==2.25.0
 requests-cache==0.6.3

From 16d092d9108618c24cea73aeef37dea2f77df34f Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:32:42 -0700
Subject: [PATCH 05/29] use wheels + main internal pypi

---
 tox.ini | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c88425b581..1b92a35519 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,10 +12,12 @@ passenv = SSH_AUTH_SOCK PAASTA_ENV DOCKER_HOST CI
 setenv =
     TZ = UTC
 deps =
+    --only-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --editable={toxinidir}
 commands =
+    --only-binary=:all:
     # these are only available at yelp so we optionally install them so that internal devs don't need to
     # manually do so (through pip or make)
     # that said, most of the time people will run make test which will use tox to install these in a
@@ -27,6 +29,7 @@ commands =
 envdir = .tox/py38-linux/
 passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR
 deps =
+    --only-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -43,6 +46,7 @@ setenv =
     PAASTA_SYSTEM_CONFIG_DIR = ./etc_paasta_playground/
     PAASTA_API_SOA_DIR = ./soa_config_playground
 deps =
+    --only-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -62,7 +66,10 @@ commands =
 
 [testenv:tests-yelpy]
 envdir = .tox/py38-linux/
+setenv =
+    PIP_INDEX_URL = http://169.254.255.254:20641/simple/
 deps =
+    --only-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -87,6 +94,7 @@ basepython = python3.8
 whitelist_externals = bash
 # one day we'll use a fully pinned venv here...
 deps =
+    --only-binary=:all:
     urllib3<2.0
     requests<2.32.0	# required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707
     cryptography<42
@@ -120,6 +128,7 @@ commands =
 changedir=example_cluster/
 passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
+    --only-binary=:all:
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
@@ -177,7 +186,9 @@ commands =
 
 [testenv:install-hooks]
 basepython = python3.8
-deps = pre-commit
+deps =
+    --only-binary=:all:
+    pre-commit
 commands = pre-commit install -f --install-hooks
 
 [flake8]

From 2173227819d1d6c44aba347e82f81b5de14cb4a4 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:33:47 -0700
Subject: [PATCH 06/29] Revert "Updating packages for the k8s_itest to pass"

This reverts commit 52b62c8b3b85ec008cc15832dbd31a57a0ee947f.
---
 requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index a974404cf7..0c0598de3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,12 +23,12 @@ cookiecutter==1.4.0
 croniter==1.3.4
 decorator==4.1.2
 docker==5.0.3
-dulwich==0.18.4
+dulwich==0.17.3
 ephemeral-port-reserve==1.1.0
 future==0.16.0
 google-auth==1.2.0
 graphviz==0.8.2
-grpcio==1.46.3
+grpcio==1.62.2
 gunicorn==19.8.1
 http-parser==0.9.0
 humanfriendly==4.18
@@ -61,7 +61,7 @@ ply==3.4
 poyo==0.4.0
 progressbar2==3.10.1
 prometheus-client==0.7.1
-protobuf==3.20.3
+protobuf==3.15.0
 py==1.5.2
 pyasn1==0.4.2
 pyasn1-modules==0.2.1
@@ -79,7 +79,7 @@ python-iptables==1.0.1
 python-utils==2.0.1
 pytimeparse==1.1.5
 pytz==2016.10
-pyyaml==6.0
+pyyaml==6.0.1
 repoze.lru==0.6
 requests==2.25.0
 requests-cache==0.6.3

From aa1df405459b35b060446fc96f37fb710eec161d Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:38:44 -0700
Subject: [PATCH 07/29] update addict

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0c0598de3a..0c2b7c4fd0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 a-sync==0.5.0
-addict==2.1.0
+addict==2.4.0
 aiohttp==3.7.4.post0
 argcomplete==0.8.1
 argparse==1.4.0

From 8a9f0d1ad3a7397c48358aee17f3b8c12ec1399f Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:45:34 -0700
Subject: [PATCH 08/29] update argcomplete

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0c2b7c4fd0..7b057f168d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 a-sync==0.5.0
 addict==2.4.0
 aiohttp==3.7.4.post0
-argcomplete==0.8.1
+argcomplete==0.8.8
 argparse==1.4.0
 arrow==1.1.0
 async-timeout==3.0.0

From e609b71138bbe54d086c5170d64b6ca1d911bdba Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:48:40 -0700
Subject: [PATCH 09/29] Revert "update argcomplete"

This reverts commit 8a9f0d1ad3a7397c48358aee17f3b8c12ec1399f.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 7b057f168d..0c2b7c4fd0 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 a-sync==0.5.0
 addict==2.4.0
 aiohttp==3.7.4.post0
-argcomplete==0.8.8
+argcomplete==0.8.1
 argparse==1.4.0
 arrow==1.1.0
 async-timeout==3.0.0

From 07dde6d3ab53109cf29d7822cae24c2db88f2db5 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:48:42 -0700
Subject: [PATCH 10/29] Revert "update addict"

This reverts commit aa1df405459b35b060446fc96f37fb710eec161d.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 0c2b7c4fd0..0c0598de3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 a-sync==0.5.0
-addict==2.4.0
+addict==2.1.0
 aiohttp==3.7.4.post0
 argcomplete==0.8.1
 argparse==1.4.0

From b79ffdfe1acc3972b981f6e0e3d6478050403f56 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 12:49:16 -0700
Subject: [PATCH 11/29] prefer binary

---
 tox.ini | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/tox.ini b/tox.ini
index 1b92a35519..1e53698117 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,12 +12,12 @@ passenv = SSH_AUTH_SOCK PAASTA_ENV DOCKER_HOST CI
 setenv =
     TZ = UTC
 deps =
-    --only-binary=:all:
+    --prefer-binary
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --editable={toxinidir}
 commands =
-    --only-binary=:all:
+    --prefer-binary
     # these are only available at yelp so we optionally install them so that internal devs don't need to
     # manually do so (through pip or make)
     # that said, most of the time people will run make test which will use tox to install these in a
@@ -29,7 +29,7 @@ commands =
 envdir = .tox/py38-linux/
 passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR
 deps =
-    --only-binary=:all:
+    --prefer-binary
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -46,7 +46,7 @@ setenv =
     PAASTA_SYSTEM_CONFIG_DIR = ./etc_paasta_playground/
     PAASTA_API_SOA_DIR = ./soa_config_playground
 deps =
-    --only-binary=:all:
+    --prefer-binary
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -69,7 +69,7 @@ envdir = .tox/py38-linux/
 setenv =
     PIP_INDEX_URL = http://169.254.255.254:20641/simple/
 deps =
-    --only-binary=:all:
+    --prefer-binary
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -94,7 +94,7 @@ basepython = python3.8
 whitelist_externals = bash
 # one day we'll use a fully pinned venv here...
 deps =
-    --only-binary=:all:
+    --prefer-binary
     urllib3<2.0
     requests<2.32.0	# required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707
     cryptography<42
@@ -128,7 +128,7 @@ commands =
 changedir=example_cluster/
 passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
-    --only-binary=:all:
+    --prefer-binary
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
@@ -187,7 +187,7 @@ commands =
 [testenv:install-hooks]
 basepython = python3.8
 deps =
-    --only-binary=:all:
+    --prefer-binary
     pre-commit
 commands = pre-commit install -f --install-hooks
 

From 3418ca28366a87b8a728c592dc3b117cfb16297a Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:05:09 -0700
Subject: [PATCH 12/29] prefer binary harder

---
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 84957612cb..66c8898210 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -45,7 +45,7 @@ WORKDIR /work
 ADD requirements.txt /work/
 RUN virtualenv /venv -ppython3.8 --no-download
 ENV PATH=/venv/bin:$PATH
-RUN pip install -r requirements.txt
+RUN pip install --prefer-binary -r requirements.txt
 ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin
 
 ADD . /work/

From cbd610f37e9b618b9ec61d2bcf8c22e3fea007fb Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:09:13 -0700
Subject: [PATCH 13/29] prefer binary this way

---
 tox.ini                                       | 16 ++++++++--------
 yelp_package/dockerfiles/itest/k8s/Dockerfile |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tox.ini b/tox.ini
index 1e53698117..caa447ac4d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,12 +12,12 @@ passenv = SSH_AUTH_SOCK PAASTA_ENV DOCKER_HOST CI
 setenv =
     TZ = UTC
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --editable={toxinidir}
 commands =
-    --prefer-binary
+    --prefer-binary=:all:
     # these are only available at yelp so we optionally install them so that internal devs don't need to
     # manually do so (through pip or make)
     # that said, most of the time people will run make test which will use tox to install these in a
@@ -29,7 +29,7 @@ commands =
 envdir = .tox/py38-linux/
 passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -46,7 +46,7 @@ setenv =
     PAASTA_SYSTEM_CONFIG_DIR = ./etc_paasta_playground/
     PAASTA_API_SOA_DIR = ./soa_config_playground
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -69,7 +69,7 @@ envdir = .tox/py38-linux/
 setenv =
     PIP_INDEX_URL = http://169.254.255.254:20641/simple/
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -94,7 +94,7 @@ basepython = python3.8
 whitelist_externals = bash
 # one day we'll use a fully pinned venv here...
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     urllib3<2.0
     requests<2.32.0	# required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707
     cryptography<42
@@ -128,7 +128,7 @@ commands =
 changedir=example_cluster/
 passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
@@ -187,7 +187,7 @@ commands =
 [testenv:install-hooks]
 basepython = python3.8
 deps =
-    --prefer-binary
+    --prefer-binary=:all:
     pre-commit
 commands = pre-commit install -f --install-hooks
 
diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 66c8898210..1786456956 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -45,7 +45,7 @@ WORKDIR /work
 ADD requirements.txt /work/
 RUN virtualenv /venv -ppython3.8 --no-download
 ENV PATH=/venv/bin:$PATH
-RUN pip install --prefer-binary -r requirements.txt
+RUN pip install --prefer-binary=:all: -r requirements.txt
 ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin
 
 ADD . /work/

From 048a1a8e1d79797f336cc91aa9fef7fb5aaa8f26 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:13:18 -0700
Subject: [PATCH 14/29] maybe?

---
 tox.ini                                       | 16 ++++++++--------
 yelp_package/dockerfiles/itest/k8s/Dockerfile |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tox.ini b/tox.ini
index caa447ac4d..8a02113be8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,12 +12,12 @@ passenv = SSH_AUTH_SOCK PAASTA_ENV DOCKER_HOST CI
 setenv =
     TZ = UTC
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --editable={toxinidir}
 commands =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     # these are only available at yelp so we optionally install them so that internal devs don't need to
     # manually do so (through pip or make)
     # that said, most of the time people will run make test which will use tox to install these in a
@@ -29,7 +29,7 @@ commands =
 envdir = .tox/py38-linux/
 passenv = PAASTA_TEST_CLUSTER KUBECONFIG PAASTA_SYSTEM_CONFIG_DIR
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -46,7 +46,7 @@ setenv =
     PAASTA_SYSTEM_CONFIG_DIR = ./etc_paasta_playground/
     PAASTA_API_SOA_DIR = ./soa_config_playground
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -69,7 +69,7 @@ envdir = .tox/py38-linux/
 setenv =
     PIP_INDEX_URL = http://169.254.255.254:20641/simple/
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
     --requirement={toxinidir}/yelp_package/extra_requirements_yelp.txt
@@ -94,7 +94,7 @@ basepython = python3.8
 whitelist_externals = bash
 # one day we'll use a fully pinned venv here...
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     urllib3<2.0
     requests<2.32.0	# required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707
     cryptography<42
@@ -128,7 +128,7 @@ commands =
 changedir=example_cluster/
 passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
@@ -187,7 +187,7 @@ commands =
 [testenv:install-hooks]
 basepython = python3.8
 deps =
-    --prefer-binary=:all:
+    --only-binary=grpcio
     pre-commit
 commands = pre-commit install -f --install-hooks
 
diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 1786456956..4d76fd0e4b 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -45,7 +45,7 @@ WORKDIR /work
 ADD requirements.txt /work/
 RUN virtualenv /venv -ppython3.8 --no-download
 ENV PATH=/venv/bin:$PATH
-RUN pip install --prefer-binary=:all: -r requirements.txt
+RUN pip install  -r requirements.txt
 ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin
 
 ADD . /work/

From 0ba8fba19c4409bcd823d8282978997467cf9b49 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:15:33 -0700
Subject: [PATCH 15/29] missed a spot

---
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 4d76fd0e4b..174f01a68f 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -45,7 +45,7 @@ WORKDIR /work
 ADD requirements.txt /work/
 RUN virtualenv /venv -ppython3.8 --no-download
 ENV PATH=/venv/bin:$PATH
-RUN pip install  -r requirements.txt
+RUN pip install --only-binary=grpcio -r requirements.txt
 ADD yelp_package/dockerfiles/itest/k8s/wait_paasta_api.sh /venv/bin
 
 ADD . /work/

From b6efc9fb4b23eabfa7a60588df2a6cb05ab190ef Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:21:02 -0700
Subject: [PATCH 16/29] upgrade??

---
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 174f01a68f..8ba52aa71a 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
-FROM ${DOCKER_REGISTRY}ubuntu:bionic
+FROM ${DOCKER_REGISTRY}ubuntu:jammy
 
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL

From c72a4ed6a6c9e27d5d266dcf9ba01102f32dae31 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:27:13 -0700
Subject: [PATCH 17/29] do we still need deadsnakes here

---
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 8ba52aa71a..07d41d5db0 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -18,11 +18,6 @@ FROM ${DOCKER_REGISTRY}ubuntu:jammy
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
-# Need Python 3.7
-RUN apt-get update > /dev/null && \
-    apt-get install -y --no-install-recommends curl software-properties-common && \
-    add-apt-repository ppa:deadsnakes/ppa
-
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         gcc \

From c4ffc3f1d063d0425c23ead2182935b7cdacd052 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:30:42 -0700
Subject: [PATCH 18/29] anotha one

---
 yelp_package/dockerfiles/itest/api/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile
index eb0bc51dee..454a8c0148 100644
--- a/yelp_package/dockerfiles/itest/api/Dockerfile
+++ b/yelp_package/dockerfiles/itest/api/Dockerfile
@@ -36,7 +36,7 @@ WORKDIR /work
 ADD requirements.txt /work/
 RUN virtualenv /venv -ppython3.8 --no-download
 ENV PATH=/venv/bin:$PATH
-RUN pip install -r requirements.txt
+RUN pip install --only-binary=grpcio -r requirements.txt
 
 COPY yelp_package/dockerfiles/xenial/mesos-slave-secret /etc/
 COPY yelp_package/dockerfiles/itest/api/mesos-cli.json yelp_package/dockerfiles/xenial/mesos-slave-secret /nail/etc/

From 447f3211616b57e3aebb8a42dffedb805ba42f05 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 13:35:27 -0700
Subject: [PATCH 19/29] more fixes

---
 yelp_package/dockerfiles/itest/api/Dockerfile | 4 ++--
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile
index 454a8c0148..7b53beeef0 100644
--- a/yelp_package/dockerfiles/itest/api/Dockerfile
+++ b/yelp_package/dockerfiles/itest/api/Dockerfile
@@ -13,9 +13,9 @@
 # limitations under the License.
 
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
-FROM ${DOCKER_REGISTRY}ubuntu:bionic
+FROM ${DOCKER_REGISTRY}ubuntu:jammy
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \
diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 07d41d5db0..76d58007df 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:jammy
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

From 0e2f1db4b5f4a1246924c3859aae12159edafef6 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 14:02:41 -0700
Subject: [PATCH 20/29] deadsnakes

---
 yelp_package/dockerfiles/itest/api/Dockerfile | 7 ++++++-
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 6 ++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile
index 7b53beeef0..774bf040bf 100644
--- a/yelp_package/dockerfiles/itest/api/Dockerfile
+++ b/yelp_package/dockerfiles/itest/api/Dockerfile
@@ -18,9 +18,14 @@ FROM ${DOCKER_REGISTRY}ubuntu:jammy
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
+RUN apt-get update -yq && \
+    apt-get install -yq \
+        # needed to add a ppa
+        software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa
+
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        software-properties-common \
         gcc \
         git \
         curl \
diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 76d58007df..27ededc90b 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -18,6 +18,12 @@ FROM ${DOCKER_REGISTRY}ubuntu:jammy
 ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
+RUN apt-get update -yq && \
+    apt-get install -yq \
+        # needed to add a ppa
+        software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa
+
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         gcc \

From 47ed5a42a99bb210907f434743733f6652976400 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 14:12:25 -0700
Subject: [PATCH 21/29] distutils is fun

---
 yelp_package/dockerfiles/itest/api/Dockerfile | 1 +
 yelp_package/dockerfiles/itest/k8s/Dockerfile | 1 +
 2 files changed, 2 insertions(+)

diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile
index 774bf040bf..43b75690b9 100644
--- a/yelp_package/dockerfiles/itest/api/Dockerfile
+++ b/yelp_package/dockerfiles/itest/api/Dockerfile
@@ -30,6 +30,7 @@ RUN apt-get update > /dev/null && \
         git \
         curl \
         python3.8-dev \
+        python3.8-distutils \
         libffi-dev \
         libssl-dev \
         libyaml-dev \
diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
index 27ededc90b..d62956ed49 100644
--- a/yelp_package/dockerfiles/itest/k8s/Dockerfile
+++ b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -30,6 +30,7 @@ RUN apt-get update > /dev/null && \
         git \
         curl \
         python3.8-dev \
+        python3.8-distutils \
         libffi-dev \
         libssl-dev \
         libyaml-dev \

From 3f6395b5d200eace057dfdcf223c284070cb1ca0 Mon Sep 17 00:00:00 2001
From: Luis Perez <luisp@yelp.com>
Date: Tue, 9 Jul 2024 14:26:40 -0700
Subject: [PATCH 22/29] cleanup

---
 tox.ini | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tox.ini b/tox.ini
index 8a02113be8..e6700a5a2b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,7 +17,6 @@ deps =
     --requirement={toxinidir}/requirements-dev.txt
     --editable={toxinidir}
 commands =
-    --only-binary=grpcio
     # these are only available at yelp so we optionally install them so that internal devs don't need to
     # manually do so (through pip or make)
     # that said, most of the time people will run make test which will use tox to install these in a
@@ -94,7 +93,6 @@ basepython = python3.8
 whitelist_externals = bash
 # one day we'll use a fully pinned venv here...
 deps =
-    --only-binary=grpcio
     urllib3<2.0
     requests<2.32.0	# required for older docker-py (<=7.0.0) to function: https://github.com/psf/requests/issues/6707
     cryptography<42
@@ -128,7 +126,6 @@ commands =
 changedir=example_cluster/
 passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
-    --only-binary=grpcio
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
@@ -187,7 +184,6 @@ commands =
 [testenv:install-hooks]
 basepython = python3.8
 deps =
-    --only-binary=grpcio
     pre-commit
 commands = pre-commit install -f --install-hooks
 

From 08184020a8556dca0e965c84a796920200644d28 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Wed, 10 Jul 2024 08:08:04 -0700
Subject: [PATCH 23/29] Revert "Updating packages for the k8s_itest to pass"

This reverts commit 52b62c8b3b85ec008cc15832dbd31a57a0ee947f.
---
 requirements.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index a974404cf7..0c0598de3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -23,12 +23,12 @@ cookiecutter==1.4.0
 croniter==1.3.4
 decorator==4.1.2
 docker==5.0.3
-dulwich==0.18.4
+dulwich==0.17.3
 ephemeral-port-reserve==1.1.0
 future==0.16.0
 google-auth==1.2.0
 graphviz==0.8.2
-grpcio==1.46.3
+grpcio==1.62.2
 gunicorn==19.8.1
 http-parser==0.9.0
 humanfriendly==4.18
@@ -61,7 +61,7 @@ ply==3.4
 poyo==0.4.0
 progressbar2==3.10.1
 prometheus-client==0.7.1
-protobuf==3.20.3
+protobuf==3.15.0
 py==1.5.2
 pyasn1==0.4.2
 pyasn1-modules==0.2.1
@@ -79,7 +79,7 @@ python-iptables==1.0.1
 python-utils==2.0.1
 pytimeparse==1.1.5
 pytz==2016.10
-pyyaml==6.0
+pyyaml==6.0.1
 repoze.lru==0.6
 requests==2.25.0
 requests-cache==0.6.3

From a9299bd2bc49084cfc67a17ebdb5639e56a029f5 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Wed, 10 Jul 2024 08:33:42 -0700
Subject: [PATCH 24/29] Removed the nerdctl regex and the adjusted ones for
 docker

---
 paasta_tools/oom_logger.py | 13 ++-----------
 tests/test_oom_logger.py   | 18 ++----------------
 2 files changed, 4 insertions(+), 27 deletions(-)

diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index 1970e60fcf..b59f58f4a8 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -104,7 +104,7 @@ def capture_oom_events_from_stdin():
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
         \s.*Task\sin\s/kubepods/(?:[a-zA-Z]+/)? # start of message; non capturing, optional group for the qos cgroup
-        pod[-\w]+/(\w{12}(?:\w{52})?)\w*\s # Match 'pod' followed by alphanumeric and hyphen characters, then capture 12 characters, optionally followed by 52 characters for container id (12 char for docker and 64 char if we're using containerd), and then zero or more word characters
+        pod[-\w]+/(\w{12})\w+\s # containerid
         killed\sas\sa*  # eom
         """,
         re.VERBOSE,
@@ -118,20 +118,12 @@ def capture_oom_events_from_stdin():
         """,
         re.VERBOSE,
     )
-    oom_regex_kubernetes_nerdctl_systemd_cgroup = re.compile(
-        r"""
-        ^(\d+)\s # timestamp
-        ([a-zA-Z0-9\-]+) # hostname
-        \s.*oom-kill:.*task_memcg=/system\.slice/.*nerdctl-(\w{64})\w*\.scope,.*$ # loosely match systemd slice and containerid
-        """,
-        re.VERBOSE,
-    )
     oom_regex_kubernetes_structured = re.compile(
         r"""
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
         \s.*oom-kill:.*task_memcg=/kubepods/(?:[a-zA-Z]+/)? # start of message; non-capturing, optional group for the qos cgroup
-        pod[-\w]+/(\w{12}(?:\w{52})?)\w*,.*$ # containerid
+        pod[-\w]+/(\w{12})\w+,.*$ # containerid
         """,
         re.VERBOSE,
     )
@@ -149,7 +141,6 @@ def capture_oom_events_from_stdin():
         oom_regex_kubernetes_structured,
         oom_regex_kubernetes_systemd_cgroup,
         oom_regex_kubernetes_containerd_systemd_cgroup,
-        oom_regex_kubernetes_nerdctl_systemd_cgroup,
     ]
 
     process_name = ""
diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index 1fbfa3d5a5..4519e9c2c8 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -220,14 +220,7 @@ def test_capture_oom_events_from_stdin_kubernetes_qos(
         test_output = []
         for a_tuple in capture_oom_events_from_stdin():
             test_output.append(a_tuple)
-        assert test_output == [
-            (
-                1500316300,
-                "dev37-devc",
-                "0e4a814eda03622476ff47871e6c397e5b8747af209b44f3b3e1c5289b0f9772",
-                "apache2",
-            )
-        ]
+        assert test_output == [(1500316300, "dev37-devc", "0e4a814eda03", "apache2")]
 
 
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
@@ -239,14 +232,7 @@ def test_capture_oom_events_from_stdin_kubernetes_structured_qos(
     test_output = []
     for a_tuple in capture_oom_events_from_stdin():
         test_output.append(a_tuple)
-    assert test_output == [
-        (
-            1500316300,
-            "dev37-devc",
-            "0e4a814eda030cdc2bb6944078eaefd3278f8f9b3a9725c4ddffb722752a2279",
-            "apache2",
-        )
-    ]
+    assert test_output == [(1500316300, "dev37-devc", "0e4a814eda03", "apache2")]
 
 
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)

From 96ce384773b6bfd5ccaced0576e9b4bef06cc787 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Mon, 15 Jul 2024 12:40:15 -0700
Subject: [PATCH 25/29] Address getting from config when its none

---
 paasta_tools/oom_logger.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index b59f58f4a8..457fe1f9e9 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -168,11 +168,12 @@ def get_container_env_as_dict(
     env_vars = {}
     if is_cri_containerd:
         config = container_inspect.get("process")
-        env = config.get("env", [])
+        env_key = "env"
     else:
         config = container_inspect.get("Config")
-        env = config.get("Env", [])
+        env_key = "Env"
     if config is not None:
+        env = config.get(env_key, [])
         for i in env:
             name, _, value = i.partition("=")
             env_vars[name] = value

From 14a16bbae829a228dba54e469a3ef02b605fe576 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Tue, 16 Jul 2024 08:21:44 -0700
Subject: [PATCH 26/29] Add a less structured regex for containerd

---
 paasta_tools/oom_logger.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/paasta_tools/oom_logger.py b/paasta_tools/oom_logger.py
index 457fe1f9e9..ee93381f5a 100644
--- a/paasta_tools/oom_logger.py
+++ b/paasta_tools/oom_logger.py
@@ -113,7 +113,17 @@ def capture_oom_events_from_stdin():
         r"""
         ^(\d+)\s # timestamp
         ([a-zA-Z0-9\-]+) # hostname
-        \s.*oom-kill:.*task_memcg=/kubepods\.slice/.* # loosely match systemd slice and containerid
+        \s.*oom-kill:.*task_memcg=/.*\.slice/.* # loosely match systemd slice and containerid
+        cri-containerd:(\w{64}).*$ # containerid
+        """,
+        re.VERBOSE,
+    )
+
+    oom_regex_kubernetes_containerd_systemd_cgroup_structured = re.compile(
+        r"""
+        ^(\d+)\s # timestamp
+        ([a-zA-Z0-9\-]+) # hostname
+        \s.*oom-kill:.*task_memcg=/kubepods\.slice/.* # match systemd slice and containerid
         cri-containerd-(\w{64}).*$ # containerid
         """,
         re.VERBOSE,
@@ -141,6 +151,7 @@ def capture_oom_events_from_stdin():
         oom_regex_kubernetes_structured,
         oom_regex_kubernetes_systemd_cgroup,
         oom_regex_kubernetes_containerd_systemd_cgroup,
+        oom_regex_kubernetes_containerd_systemd_cgroup_structured,
     ]
 
     process_name = ""

From 4132b65489fc462e36793a0313851d8a2d22cd17 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Tue, 27 Aug 2024 05:13:11 -0700
Subject: [PATCH 27/29] Added two tests to test the regex if its working

---
 tests/test_oom_logger.py | 75 ++++++++++++++++++++++++++++++++++++----
 1 file changed, 69 insertions(+), 6 deletions(-)

diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index 4519e9c2c8..51b45082ce 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -21,7 +21,6 @@
 from paasta_tools.oom_logger import log_to_clog
 from paasta_tools.oom_logger import LogLine
 from paasta_tools.oom_logger import main
-from paasta_tools.oom_logger import parse_args
 from paasta_tools.oom_logger import send_sfx_event
 
 
@@ -121,6 +120,39 @@ def sys_stdin_kubernetes_structured_burstable_systemd_cgroup():
     ]
 
 
+@pytest.fixture
+def sys_stdin_kubernetes_containerd_systemd_cgroup_structured():
+    return [
+        "some random line1\n",
+        "1720128512 dev208-uswest1adevc [ 7195.442797] python3 invoked oom-killer: "
+        "gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=999\n",
+        "some random line2\n",
+        "1720128512 dev208-uswest1adevc [ 7195.442928] oom-kill:constraint=CONSTRAINT_MEMCG,"
+        "cpuset=cri-containerd-e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c.scope,"
+        "mems_allowed=0,oom_memcg=/kubepods.slice/kubepods-burstable.slice/"
+        "kubepods-burstable-pod08768c36_163c_40e5_8e49_09cf42ff5046.slice,"
+        "task_memcg=/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod08768c36_163c_40e5_8e49_09cf42ff5046.slice/"
+        "cri-containerd-e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c.scope,task=python3,pid=485850,uid=33\n",
+    ]
+
+
+@pytest.fixture
+def sys_stdin_kubernetes_containerd_systemd_cgroup():
+    return [
+        "some random line1\n",
+        "1720128512 dev208-uswest1adevc [42201.484624] python3 invoked oom-killer: "
+        "gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=999\n",
+        "some random line2\n",
+        "1720128512 dev208-uswest1adevc [42201.484749] oom-kill:constraint=CONSTRAINT_MEMCG,"
+        "nodemask=(null),cpuset=kubepods-burstable-pod73331cbb_9b96_4a62_9702_46a56ad49dd0.slice:"
+        "cri-containerd:52f9ece9bcf929a08951aa3b4312fbec50890d82b58988f91a0aa9dc96ebc199,"
+        "mems_allowed=0,oom_memcg=/system.slice/kubepods-burstable-pod73331cbb_9b96_4a62_9702_46a56ad49dd0.slice:"
+        "cri-containerd:52f9ece9bcf929a08951aa3b4312fbec50890d82b58988f91a0aa9dc96ebc199,"
+        "task_memcg=/system.slice/kubepods-burstable-pod73331cbb_9b96_4a62_9702_46a56ad49dd0.slice:"
+        "cri-containerd:52f9ece9bcf929a08951aa3b4312fbec50890d82b58988f91a0aa9dc96ebc199,task=python3,pid=4190418,uid=33\n",
+    ]
+
+
 @pytest.fixture
 def sys_stdin_process_name_with_slashes():
     return [
@@ -247,6 +279,42 @@ def test_capture_oom_events_from_stdin_kubernetes_structured_burstable_systemd_c
     assert test_output == [(1500316300, "dev37-devc", "e7ba37bd3708", "apache2")]
 
 
+@patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
+def test_capture_oom_events_from_stdin_kubernetes_containerd_systemd_cgroup(
+    mock_sys_stdin,
+    sys_stdin_kubernetes_containerd_systemd_cgroup,
+):
+    mock_sys_stdin.readline.side_effect = sys_stdin_kubernetes_containerd_systemd_cgroup
+    test_output = [a_tuple for a_tuple in capture_oom_events_from_stdin()]
+    assert test_output == [
+        (
+            1720128512,
+            "dev208-uswest1adevc",
+            "52f9ece9bcf929a08951aa3b4312fbec50890d82b58988f91a0aa9dc96ebc199",
+            "python3",
+        )
+    ]
+
+
+@patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
+def test_capture_oom_events_from_stdin_kubernetes_containerd_systemd_cgroup_structured(
+    mock_sys_stdin,
+    sys_stdin_kubernetes_containerd_systemd_cgroup_structured,
+):
+    mock_sys_stdin.readline.side_effect = (
+        sys_stdin_kubernetes_containerd_systemd_cgroup_structured
+    )
+    test_output = [a_tuple for a_tuple in capture_oom_events_from_stdin()]
+    assert test_output == [
+        (
+            1720128512,
+            "dev208-uswest1adevc",
+            "e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c",
+            "python3",
+        )
+    ]
+
+
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
 def test_capture_oom_events_from_stdin_with_slashes(
     mock_sys_stdin, sys_stdin_process_name_with_slashes
@@ -345,11 +413,6 @@ def test_send_sfx_event(mock_get_instance_config):
         assert mock_meteorite.create_counter.return_value.count.call_count == 1
 
 
-@patch("paasta_tools.oom_logger.argparse", autospec=True)
-def test_parse_args(mock_argparse):
-    assert parse_args() == mock_argparse.ArgumentParser.return_value.parse_args()
-
-
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
 @patch("paasta_tools.oom_logger.clog", autospec=True)
 @patch("paasta_tools.oom_logger.send_sfx_event", autospec=True)

From f3c13bc2742bbfbb4888e24d7f14ace927cdbe93 Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Tue, 27 Aug 2024 08:34:08 -0700
Subject: [PATCH 28/29] Adding a unit test for testing main with
 containerd=true

---
 tests/test_oom_logger.py | 77 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index 51b45082ce..fd4f1cc582 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -14,6 +14,7 @@
 import json
 
 import pytest
+from mock import MagicMock
 from mock import Mock
 from mock import patch
 
@@ -211,6 +212,20 @@ def docker_inspect():
     }
 
 
+@pytest.fixture
+def containerd_inspect():
+    return {
+        "process": {
+            "env": [
+                "PAASTA_SERVICE=fake_service",
+                "PAASTA_INSTANCE=fake_instance",
+                "PAASTA_RESOURCE_MEM=512",
+                "MESOS_CONTAINER_NAME=mesos-a04c14a6-83ea-4047-a802-92b850b1624e",
+            ]
+        }
+    }
+
+
 @pytest.fixture
 def log_line():
     return LogLine(
@@ -226,6 +241,21 @@ def log_line():
     )
 
 
+@pytest.fixture
+def log_line_containerd():
+    return LogLine(
+        timestamp=1720128512,
+        hostname="dev208-uswest1adevc",
+        container_id="e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c",
+        cluster="fake_cluster",
+        service="fake_service",
+        instance="fake_instance",
+        process_name="python3",
+        mesos_container_id="mesos-a04c14a6-83ea-4047-a802-92b850b1624e",
+        mem_limit="512",
+    )
+
+
 @patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
 def test_capture_oom_events_from_stdin(mock_sys_stdin, sys_stdin):
     mock_sys_stdin.readline.side_effect = sys_stdin
@@ -449,3 +479,50 @@ def test_main(
     mock_send_sfx_event.assert_called_once_with(
         "fake_service", "fake_instance", "fake_cluster"
     )
+
+
+@patch("paasta_tools.oom_logger.sys.stdin", autospec=True)
+@patch("paasta_tools.oom_logger.clog", autospec=True)
+@patch("paasta_tools.oom_logger.send_sfx_event", autospec=True)
+@patch("paasta_tools.oom_logger.load_system_paasta_config", autospec=True)
+@patch("paasta_tools.oom_logger.log_to_clog", autospec=True)
+@patch("paasta_tools.oom_logger.log_to_paasta", autospec=True)
+@patch("paasta_tools.oom_logger.parse_args", autospec=True)
+@patch("paasta_tools.oom_logger.get_containerd_container", autospec=True)
+@patch("paasta_tools.oom_logger.json.loads", autospec=True)
+def test_main_containerd(
+    mock_json_loads,
+    mock_get_containerd_container,
+    mock_parse_args,
+    mock_log_to_paasta,
+    mock_log_to_clog,
+    mock_load_system_paasta_config,
+    mock_send_sfx_event,
+    mock_clog,
+    mock_sys_stdin,
+    sys_stdin_kubernetes_containerd_systemd_cgroup_structured,
+    log_line_containerd,
+    containerd_inspect,
+):
+
+    mock_sys_stdin.readline.side_effect = (
+        sys_stdin_kubernetes_containerd_systemd_cgroup_structured
+    )
+    mock_parse_args.return_value.containerd = True
+
+    mock_container_info = MagicMock()
+    mock_container_info.spec.value.decode.return_value = str(containerd_inspect)
+
+    mock_get_containerd_container.return_value = mock_container_info
+    mock_json_loads.return_value = containerd_inspect
+
+    mock_load_system_paasta_config.return_value.get_cluster.return_value = (
+        "fake_cluster"
+    )
+
+    main()
+    mock_log_to_paasta.assert_called_once_with(log_line_containerd)
+    mock_log_to_clog.assert_called_once_with(log_line_containerd)
+    mock_send_sfx_event.assert_called_once_with(
+        "fake_service", "fake_instance", "fake_cluster"
+    )

From 416f9b98f3e092caa1a810429461af2b0fcedf2d Mon Sep 17 00:00:00 2001
From: Eman Elsabban <emanelsabban@yelp.com>
Date: Wed, 28 Aug 2024 10:49:52 -0700
Subject: [PATCH 29/29] Addressing more reviews

---
 tests/test_oom_logger.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tests/test_oom_logger.py b/tests/test_oom_logger.py
index fd4f1cc582..16821a528a 100644
--- a/tests/test_oom_logger.py
+++ b/tests/test_oom_logger.py
@@ -125,10 +125,10 @@ def sys_stdin_kubernetes_structured_burstable_systemd_cgroup():
 def sys_stdin_kubernetes_containerd_systemd_cgroup_structured():
     return [
         "some random line1\n",
-        "1720128512 dev208-uswest1adevc [ 7195.442797] python3 invoked oom-killer: "
+        "1720128512 dev37-devc [ 7195.442797] python3 invoked oom-killer: "
         "gfp_mask=0xcc0(GFP_KERNEL), order=0, oom_score_adj=999\n",
         "some random line2\n",
-        "1720128512 dev208-uswest1adevc [ 7195.442928] oom-kill:constraint=CONSTRAINT_MEMCG,"
+        "1720128512 dev37-devc [ 7195.442928] oom-kill:constraint=CONSTRAINT_MEMCG,"
         "cpuset=cri-containerd-e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c.scope,"
         "mems_allowed=0,oom_memcg=/kubepods.slice/kubepods-burstable.slice/"
         "kubepods-burstable-pod08768c36_163c_40e5_8e49_09cf42ff5046.slice,"
@@ -220,7 +220,6 @@ def containerd_inspect():
                 "PAASTA_SERVICE=fake_service",
                 "PAASTA_INSTANCE=fake_instance",
                 "PAASTA_RESOURCE_MEM=512",
-                "MESOS_CONTAINER_NAME=mesos-a04c14a6-83ea-4047-a802-92b850b1624e",
             ]
         }
     }
@@ -245,13 +244,13 @@ def log_line():
 def log_line_containerd():
     return LogLine(
         timestamp=1720128512,
-        hostname="dev208-uswest1adevc",
+        hostname="dev37-devc",
         container_id="e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c",
         cluster="fake_cluster",
         service="fake_service",
         instance="fake_instance",
         process_name="python3",
-        mesos_container_id="mesos-a04c14a6-83ea-4047-a802-92b850b1624e",
+        mesos_container_id="mesos-null",
         mem_limit="512",
     )
 
@@ -338,7 +337,7 @@ def test_capture_oom_events_from_stdin_kubernetes_containerd_systemd_cgroup_stru
     assert test_output == [
         (
             1720128512,
-            "dev208-uswest1adevc",
+            "dev37-devc",
             "e216d2f1e6c625d363c71edb6b3cbab5a9e1b447641b61028d0b94b077adf27c",
             "python3",
         )