Merge branch 'master' of github.com:Yelp/Tron into u/emanelsabban/TRO…

…N-2161

debian/changelog

@@ -1,3 +1,13 @@
+tron (1.30.0) jammy; urgency=medium
+
+  * 1.30.0 tagged with 'make release'
+    Commit: Adding yelp_clog S3LogsReader (#949)  * Upgrade yelp
+    scribereader deps  * Enable S3LogsReader for action run logs  *
+    Upgrade boto requirements  * Upgrade mypy and update type ignores  *
+    Pin more internal requirements
+
+ -- Yaroslav Liakhovskyi <[email protected]>  Mon, 29 Apr 2024 04:57:37 -0700
+
 tron (1.29.5) jammy; urgency=medium
 
   * 1.29.5 tagged with 'make release'

mypy.ini

@@ -12,3 +12,9 @@ warn_unused_ignores = True
 
 [mypy-clusterman_metrics.*]
 ignore_missing_imports = True
+
+[mypy-clog.*]
+ignore_missing_imports = True
+
+[mypy-scribereader.*]
+ignore_missing_imports = True

requirements-dev-minimal.txt

@@ -7,3 +7,6 @@ pylint
 pytest
 pytest-asyncio
 requirements-tools
+types-PyYAML
+types-requests<2.31.0.7 # newer types-requests requires urllib3>=2
+types-simplejson

requirements-dev.txt

@@ -10,8 +10,8 @@ iniconfig==1.1.1
 isort==4.3.18
 lazy-object-proxy==1.9.0
 mccabe==0.7.0
-mypy==0.812
-mypy-extensions==0.4.3
+mypy==1.9.0
+mypy-extensions==1.0.0
 nodeenv==1.3.3
 packaging==19.2
 platformdirs==2.5.2
@@ -28,5 +28,8 @@ requirements-tools==1.2.1
 toml==0.10.2
 tomli==2.0.1
 tomlkit==0.11.6
-typed-ast==1.4.0
+types-PyYAML==6.0.12
+types-requests==2.31.0.5
+types-simplejson==3.19.0.20240310
+types-urllib3==1.26.25.14
 virtualenv==20.17.1

requirements.txt

@@ -7,8 +7,8 @@ aws-sam-translator==1.15.1
 aws-xray-sdk==2.4.2
 backcall==0.1.0
 boto==2.49.0
-boto3==1.26.85
-botocore==1.29.86
+boto3==1.34.80
+botocore==1.34.80
 bsddb3==6.2.7
 cachetools==4.2.1
 certifi==2022.12.7
@@ -75,7 +75,7 @@ requests==2.25.0
 requests-oauthlib==1.2.0
 responses==0.10.6
 rsa==4.9
-s3transfer==0.6.0
+s3transfer==0.10.1
 setuptools==65.5.1
 six==1.15.0
 sshpubkeys==3.1.0

tests/utils/scribereader_test.py

@@ -3,12 +3,81 @@
 
 import pytest
 
+import tron.utils.scribereader
 from tron.utils.scribereader import read_log_stream_for_action_run
 
 try:
     import scribereader  # noqa: F401
+    from clog.readers import S3LogsReader  # noqa: F401
 except ImportError:
-    pytest.skip("scribereader not available, skipping tests", allow_module_level=True)
+    pytest.skip("yelp logs readers not available, skipping tests", allow_module_level=True)
+
+
+# used for an explicit patch of staticconf.read return value for an arbitrary namespace
+def static_conf_patch(args):
+    return lambda arg, namespace, default=None: args.get(arg)
+
+
+def test_read_log_stream_for_action_run_not_available():
+    with mock.patch("tron.utils.scribereader.scribereader_available", False), mock.patch(
+        "tron.utils.scribereader.s3reader_available", False
+    ):
+        output = tron.utils.scribereader.read_log_stream_for_action_run(
+            "namespace.job.1234.action",
+            component="stdout",
+            min_date=datetime.datetime.now(),
+            max_date=datetime.datetime.now(),
+            paasta_cluster="fake",
+        )
+    assert "unable to display logs" in output[0]
+
+
+def test_read_log_stream_for_action_run_yelp_clog():
+    with mock.patch(
+        "staticconf.read",
+        autospec=True,
+        side_effect=static_conf_patch({"logging.use_s3_reader": True, "logging.max_lines_to_display": 1000}),
+    ), mock.patch("tron.config.static_config.build_configuration_watcher", autospec=True,), mock.patch(
+        "tron.config.static_config.load_yaml_file",
+        autospec=True,
+    ), mock.patch(
+        "tron.utils.scribereader.S3LogsReader", autospec=True
+    ) as mock_s3_reader:
+
+        mock_s3_reader.return_value.get_log_reader.return_value = iter(
+            [
+                """{
+                "tron_run_number": 1234,
+                "component": "stdout",
+                "message": "line 1",
+                "timestamp": "2021-01-02T18:10:09.169421619Z",
+                "cluster": "fake"
+            }""",
+                """{
+                "tron_run_number": 1234,
+                "component": "stdout",
+                "message": "line 2",
+                "timestamp": "2021-01-02T18:11:09.169421619Z",
+                "cluster": "fake"
+            }""",
+                """{
+                "tron_run_number": 1234,
+                "component": "stderr",
+                "message": "line 3",
+                "timestamp": "2021-01-02T18:12:09.169421619Z",
+                "cluster": "fake"
+            }""",
+            ]
+        )
+
+        output = read_log_stream_for_action_run(
+            "namespace.job.1234.action",
+            component="stdout",
+            min_date=datetime.datetime.now(),
+            max_date=datetime.datetime.now(),
+            paasta_cluster="fake",
+        )
+    assert output == ["line 1", "line 2"]
 
 
 def test_read_log_stream_for_action_run_min_date_and_max_date_today():
@@ -35,7 +104,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_today():
         "tron.config.static_config.build_configuration_watcher",
         autospec=True,
     ), mock.patch(
-        "staticconf.read", autospec=True, return_value=1000
+        "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000})
     ), mock.patch(
         "tron.config.static_config.load_yaml_file",
         autospec=True,
@@ -111,7 +180,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_different_days():
         "tron.config.static_config.build_configuration_watcher",
         autospec=True,
     ), mock.patch(
-        "staticconf.read", autospec=True, return_value=1000
+        "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000})
     ), mock.patch(
         "tron.config.static_config.load_yaml_file",
         autospec=True,
@@ -209,7 +278,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_in_past():
         "tron.config.static_config.build_configuration_watcher",
         autospec=True,
     ), mock.patch(
-        "staticconf.read", autospec=True, return_value=1000
+        "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000})
     ), mock.patch(
         "tron.config.static_config.load_yaml_file",
         autospec=True,
@@ -275,7 +344,7 @@ def test_read_log_stream_for_action_run_min_date_and_max_date_for_long_output():
         "tron.config.static_config.build_configuration_watcher",
         autospec=True,
     ), mock.patch(
-        "staticconf.read", autospec=True, return_value=1000
+        "staticconf.read", autospec=True, side_effect=static_conf_patch({"logging.max_lines_to_display": 1000})
     ), mock.patch(
         "tron.config.static_config.load_yaml_file",
         autospec=True,

tron/__init__.py

@@ -1,4 +1,4 @@
-__version_info__ = (1, 29, 5)
+__version_info__ = (1, 30, 0)
 __version__ = ".".join("%s" % v for v in __version_info__)
 __author__ = "Yelp <[email protected]>"
 __credits__ = [

tron/commands/backfill.py

@@ -112,7 +112,7 @@ async def create(self) -> Optional[str]:
             match = re.match(r"^Created JobRun:([-.\w]+)$", result)
 
             if match:
-                self.run_name = match.groups(0)[0]
+                self.run_name = match.groups(0)[0]  # type: ignore[assignment] # mypy wrongly identifies self.run_name type as "Union[str, int]"
                 self.run_state = ActionRun.STARTING
                 print(f"Job run '{self.run_name}' for {self.run_time_str} created")
             else:

tron/commands/display.py

@@ -269,7 +269,7 @@ class DisplayJobRuns(TableDisplay):
     colors = {
         "id": partial(Color.set, "yellow"),
         "state": add_color_for_state,
-        "manual": lambda value: Color.set("cyan" if value else None, value),  # type: ignore  # can't type a lambda
+        "manual": lambda value: Color.set("cyan" if value else None, value),
     }
 
     def format_value(self, field_idx, value):

tron/core/actionrun.py

@@ -19,7 +19,7 @@
 from tron.actioncommand import ActionCommand
 from tron.actioncommand import NoActionRunnerFactory
 from tron.actioncommand import SubprocessActionRunnerFactory
-from tron.bin.action_runner import build_environment  # type: ignore # mypy can't find library stub
+from tron.bin.action_runner import build_environment
 from tron.bin.action_runner import build_labels
 from tron.config.config_utils import StringFormatter
 from tron.config.schema import ExecutorTypes