Fix alert for no new flaky tests spamming (#5483)

![image](https://github.com/user-attachments/assets/2b81448d-32f3-4a54-909b-40cf2f9a9dcc)
Made a ton of issues for some reason

This change checks for an open issue made in the last 7 days. If it sees
one, it doesn't make a new one

tools/torchci/check_alerts.py

@@ -6,7 +6,7 @@
 import re
 import urllib.parse
 from collections import defaultdict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from difflib import SequenceMatcher
 from typing import Any, Dict, List, Set, Tuple
 
@@ -497,14 +497,10 @@ def classify_jobs(
     return jobs_to_alert_on, flaky_jobs
 
 
-def handle_flaky_tests_alert(existing_alerts: List[Dict]) -> Dict:
-    if (
-        not existing_alerts
-        or datetime.fromisoformat(
-            existing_alerts[0]["createdAt"].replace("Z", "+00:00")
-        ).date()
-        != datetime.today().date()
-    ):
+def handle_flaky_tests_alert(
+    existing_alerts: List[Dict], dry_run: bool = False
+) -> Dict:
+    if not existing_alerts:
         from_date = (
             datetime.today() - timedelta(days=FLAKY_TESTS_SEARCH_PERIOD_DAYS)
         ).strftime("%Y-%m-%d")
@@ -516,7 +512,7 @@ def handle_flaky_tests_alert(existing_alerts: List[Dict]) -> Dict:
             num_issues_with_flaky_tests_lables,
         )
         if num_issues_with_flaky_tests_lables == 0:
-            return create_issue(generate_no_flaky_tests_issue(), False)
+            return create_issue(generate_no_flaky_tests_issue(), dry_run=dry_run)
 
     print("No new alert for flaky tests bots.")
     return None
@@ -598,7 +594,17 @@ def check_for_no_flaky_tests_alert(repo: str, branch: str):
     existing_no_flaky_tests_alerts = fetch_alerts(
         labels=[NO_FLAKY_TESTS_LABEL],
     )
-    handle_flaky_tests_alert(existing_no_flaky_tests_alerts)
+    open_alerts = [
+        alert for alert in existing_no_flaky_tests_alerts if not alert["closed"]
+    ]
+    recent_open_alerts = [
+        existing_alert
+        for existing_alert in open_alerts
+        if datetime.now(timezone.utc)
+        - datetime.fromisoformat(existing_alert["createdAt"].replace("Z", "+00:00"))
+        < timedelta(days=7)
+    ]
+    handle_flaky_tests_alert(recent_open_alerts)
 
 
 def parse_args() -> argparse.Namespace:

tools/torchci/tests/test_check_alerts.py

@@ -1,9 +1,10 @@
 import json
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from unittest import main, TestCase
 from unittest.mock import patch
 
 from torchci.check_alerts import (
+    check_for_no_flaky_tests_alert,
     fetch_alerts_filter,
     filter_job_names,
     gen_update_comment,
@@ -197,27 +198,75 @@ def test_handle_flaky_tests_alert(
         mock_date.today.return_value = datetime(2022, 10, 10)
         mock_get_num_issues_with_label.return_value = 5
 
-        res = handle_flaky_tests_alert([])
+        # No open alert issues, but also there are some flaky tests
+        res = handle_flaky_tests_alert([], dry_run=True)
         self.assertIsNone(res)
 
         existing_alerts = [
             {"createdAt": "2022-10-10T13:41:09Z"},
             {"createdAt": "2022-10-08T14:41:09Z"},
         ]
-        res = handle_flaky_tests_alert(existing_alerts)
-        self.assertIsNone(res)
 
-        existing_alerts = [
-            {"createdAt": "2022-10-09T13:41:09Z"},
-            {"createdAt": "2022-10-08T14:41:09Z"},
-        ]
-        res = handle_flaky_tests_alert(existing_alerts)
+        # Open alert issue, and there are some flaky tests
+        res = handle_flaky_tests_alert(existing_alerts, dry_run=True)
         self.assertIsNone(res)
 
+        # No open alert issue, and no flaky tests
         mock_get_num_issues_with_label.return_value = 0
-        res = handle_flaky_tests_alert(existing_alerts)
+        res = handle_flaky_tests_alert([], dry_run=True)
         self.assertDictEqual(res, mock_issue)
 
+        # Open alert issue, and no flaky tests
+        mock_get_num_issues_with_label.return_value = 0
+        res = handle_flaky_tests_alert(existing_alerts, dry_run=True)
+        self.assertIsNone(res)
+
+    @patch("torchci.check_alerts.fetch_alerts")
+    @patch("torchci.check_alerts.handle_flaky_tests_alert")
+    def test_check_for_no_flaky_tests_alert(
+        self,
+        mock_handle_flaky_tests_alert,
+        mock_fetch_alerts,
+    ):
+        # Issue is open but created too long ago
+        mock_fetch_alerts.return_value = [
+            {
+                "closed": False,
+                "createdAt": (
+                    datetime.now(timezone.utc) - timedelta(days=7.1)
+                ).isoformat(),
+            }
+        ]
+        check_for_no_flaky_tests_alert("dummy repo", "dummy branch")
+        first_argument = mock_handle_flaky_tests_alert.call_args.args[0]
+        self.assertListEqual(first_argument, [])
+
+        # Issue is open and recent
+        mock_fetch_alerts.return_value = [
+            {
+                "closed": False,
+                "createdAt": (
+                    datetime.now(timezone.utc) - timedelta(days=6.9)
+                ).isoformat(),
+            }
+        ]
+        check_for_no_flaky_tests_alert("dummy repo", "dummy branch")
+        first_argument = mock_handle_flaky_tests_alert.call_args.args[0]
+        self.assertDictEqual(first_argument[0], mock_fetch_alerts.return_value[0])
+
+        # Issue is closed and recent
+        mock_fetch_alerts.return_value = [
+            {
+                "closed": True,
+                "createdAt": (
+                    datetime.now(timezone.utc) - timedelta(days=6.9)
+                ).isoformat(),
+            }
+        ]
+        check_for_no_flaky_tests_alert("dummy repo", "dummy branch")
+        first_argument = mock_handle_flaky_tests_alert.call_args.args[0]
+        self.assertListEqual(first_argument, [])
+
     # test filter job names
     def test_job_filter(self):
         job_names = [