[Backend] Add disable logs for challenge phases (#4015)

* Add disable logs to challenge phases * Add disable logs to phase create serializer
Cloud-CV · Jul 12, 2023 · 5301068 · 5301068
1 parent 7faa96f
commit 5301068
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 62 deletions.
diff --git a/apps/challenges/migrations/0097_challengephase_disable_logs.py b/apps/challenges/migrations/0097_challengephase_disable_logs.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.20 on 2023-07-11 07:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('challenges', '0096_challenge_meta_order_by_title'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='challengephase',
+            name='disable_logs',
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/apps/challenges/models.py b/apps/challenges/models.py
@@ -324,6 +324,7 @@ def __init__(self, *args, **kwargs):
     default_submission_meta_attributes = JSONField(
         default=None, blank=True, null=True
     )
+    disable_logs = models.BooleanField(default=False)
 
     class Meta:
         app_label = "challenges"

diff --git a/apps/challenges/serializers.py b/apps/challenges/serializers.py
@@ -116,6 +116,7 @@ class Meta:
             "default_submission_meta_attributes",
             "allowed_email_ids",
             "is_submission_public",
+            "disable_logs",
         )
 
 
@@ -376,6 +377,7 @@ class Meta:
             "allowed_submission_file_types",
             "default_submission_meta_attributes",
             "allowed_email_ids",
+            "disable_logs",
         )
 
 

diff --git a/scripts/workers/code_upload_submission_worker.py b/scripts/workers/code_upload_submission_worker.py
@@ -576,6 +576,7 @@ def update_failed_jobs_and_send_logs(
     message,
     queue_name,
     is_remote,
+    disable_logs
 ):
     clean_submission = False
     code_upload_environment_error = "Submission Job Failed."
@@ -590,47 +591,51 @@ def update_failed_jobs_and_send_logs(
                 label_selector=pod_label_selector,
                 timeout_seconds=10,
             )
-            # Prevents monitoring when Job created with pending pods state (not assigned to node)
-            if pods_list.items[0].status.container_statuses:
-                container_state_map = {}
-                for container in pods_list.items[0].status.container_statuses:
-                    container_state_map[container.name] = container.state
-                for (
-                    container_name,
-                    container_state,
-                ) in container_state_map.items():
-                    if container_name in ["agent", "submission", "environment"]:
-                        if container_state.terminated is not None:
-                            pod_name = pods_list.items[0].metadata.name
-                            try:
-                                pod_log_response = core_v1_api_instance.read_namespaced_pod_log(
-                                    name=pod_name,
-                                    namespace="default",
-                                    _return_http_data_only=True,
-                                    _preload_content=False,
-                                    container=container_name,
-                                )
-                                pod_log = pod_log_response.data.decode(
-                                    "utf-8"
-                                )
-                                pod_log = pod_log[-10000:]
-                                clean_submission = True
-                                if container_name == "environment":
-                                    code_upload_environment_error = pod_log
-                                else:
-                                    submission_error = pod_log
-                            except client.rest.ApiException as e:
-                                logger.exception(
-                                    "Exception while reading Job logs {}".format(
-                                        e
-                                    )
-                                )
+            if disable_logs:
+                code_upload_environment_error = None
+                submission_error = None
             else:
-                logger.info(
-                    "Job pods in pending state, waiting for node assignment for submission {}".format(
-                        submission_pk
+                # Prevents monitoring when Job created with pending pods state (not assigned to node)
+                if pods_list.items[0].status.container_statuses:
+                    container_state_map = {}
+                    for container in pods_list.items[0].status.container_statuses:
+                        container_state_map[container.name] = container.state
+                    for (
+                        container_name,
+                        container_state,
+                    ) in container_state_map.items():
+                        if container_name in ["agent", "submission", "environment"]:
+                            if container_state.terminated is not None:
+                                pod_name = pods_list.items[0].metadata.name
+                                try:
+                                    pod_log_response = core_v1_api_instance.read_namespaced_pod_log(
+                                        name=pod_name,
+                                        namespace="default",
+                                        _return_http_data_only=True,
+                                        _preload_content=False,
+                                        container=container_name,
+                                    )
+                                    pod_log = pod_log_response.data.decode(
+                                        "utf-8"
+                                    )
+                                    pod_log = pod_log[-10000:]
+                                    clean_submission = True
+                                    if container_name == "environment":
+                                        code_upload_environment_error = pod_log
+                                    else:
+                                        submission_error = pod_log
+                                except client.rest.ApiException as e:
+                                    logger.exception(
+                                        "Exception while reading Job logs {}".format(
+                                            e
+                                        )
+                                    )
+                else:
+                    logger.info(
+                        "Job pods in pending state, waiting for node assignment for submission {}".format(
+                            submission_pk
+                        )
                     )
-                )
         else:
             logger.exception(
                 "Exception while reading Job {}, does not exist.".format(
@@ -740,6 +745,8 @@ def main():
             submission_pk = message_body.get("submission_pk")
             challenge_pk = message_body.get("challenge_pk")
             phase_pk = message_body.get("phase_pk")
+            challenge_phase = evalai.get_challenge_phase_by_pk(challenge_pk, phase_pk)
+            disable_logs = challenge_phase.get("disable_logs")
             submission = evalai.get_submission_by_pk(submission_pk)
             if submission:
                 if (
@@ -789,6 +796,7 @@ def main():
                         message,
                         QUEUE_NAME,
                         is_remote,
+                        disable_logs,
                     )
                 else:
                     logger.info(

diff --git a/scripts/workers/submission_worker.py b/scripts/workers/submission_worker.py
@@ -454,18 +454,19 @@ def run_submission(
                     SUBMISSION_LOGS_PREFIX, submission.id
                 )
             )
-            with stdout_redirect(
-                MultiOut(stdout, sys.__stdout__)
-            ) as new_stdout, stderr_redirect(  # noqa
-                MultiOut(stderr, sys.__stderr__)
-            ) as new_stderr:  # noqa
-                submission_output = EVALUATION_SCRIPTS[challenge_id].evaluate(
-                    annotation_file_path,
-                    user_annotation_file_path,
-                    challenge_phase.codename,
-                    submission_metadata=submission_serializer.data,
-                )
-                return
+            if not challenge_phase.disable_logs:
+                with stdout_redirect(
+                    MultiOut(stdout, sys.__stdout__)
+                ) as new_stdout, stderr_redirect(  # noqa
+                    MultiOut(stderr, sys.__stderr__)
+                ) as new_stderr:  # noqa
+                    submission_output = EVALUATION_SCRIPTS[challenge_id].evaluate(
+                        annotation_file_path,
+                        user_annotation_file_path,
+                        challenge_phase.codename,
+                        submission_metadata=submission_serializer.data,
+                    )
+            return
         except Exception:
             stderr.write(traceback.format_exc())
             stderr.close()
@@ -491,17 +492,18 @@ def run_submission(
     # call `main` from globals and set `status` to running and hence `started_at`
     try:
         successful_submission_flag = True
-        with stdout_redirect(
-            MultiOut(stdout, sys.__stdout__)
-        ) as new_stdout, stderr_redirect(  # noqa
-            MultiOut(stderr, sys.__stderr__)
-        ) as new_stderr:  # noqa
-            submission_output = EVALUATION_SCRIPTS[challenge_id].evaluate(
-                annotation_file_path,
-                user_annotation_file_path,
-                challenge_phase.codename,
-                submission_metadata=submission_serializer.data,
-            )
+        if not challenge_phase.disable_logs:
+            with stdout_redirect(
+                MultiOut(stdout, sys.__stdout__)
+            ) as new_stdout, stderr_redirect(  # noqa
+                MultiOut(stderr, sys.__stderr__)
+            ) as new_stderr:  # noqa
+                submission_output = EVALUATION_SCRIPTS[challenge_id].evaluate(
+                    annotation_file_path,
+                    user_annotation_file_path,
+                    challenge_phase.codename,
+                    submission_metadata=submission_serializer.data,
+                )
         """
         A submission will be marked successful only if it is of the format
             {

diff --git a/tests/unit/challenges/test_serializers.py b/tests/unit/challenges/test_serializers.py
@@ -225,6 +225,7 @@ def test_challenge_phase_create_serializer(self):
                     "allowed_submission_file_types",
                     "default_submission_meta_attributes",
                     "allowed_email_ids",
+                    "disable_logs",
                 ]
             ),
         )
@@ -309,6 +310,7 @@ def test_challenge_phase_create_serializer_wihout_max_submissions_per_month(
                     "allowed_submission_file_types",
                     "default_submission_meta_attributes",
                     "allowed_email_ids",
+                    "disable_logs",
                 ]
             ),
         )
@@ -389,6 +391,7 @@ def test_challenge_phase_create_serializer_without_max_concurrent_submissions_al
                     "allowed_submission_file_types",
                     "default_submission_meta_attributes",
                     "allowed_email_ids",
+                    "disable_logs",
                 ]
             ),
         )

diff --git a/tests/unit/challenges/test_views.py b/tests/unit/challenges/test_views.py
@@ -2463,6 +2463,7 @@ def test_get_challenge_phase(self):
                 "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
                 "is_submission_public": self.challenge_phase.is_submission_public,
+                "disable_logs": self.challenge_phase.disable_logs,
             },
             {
                 "id": self.private_challenge_phase.id,
@@ -2491,6 +2492,7 @@ def test_get_challenge_phase(self):
                 "default_submission_meta_attributes": self.private_challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
                 "is_submission_public": self.challenge_phase.is_submission_public,
+                "disable_logs": self.challenge_phase.disable_logs,
             },
         ]
 
@@ -2527,6 +2529,7 @@ def test_get_challenge_phase_when_user_is_not_authenticated(self):
                 "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
                 "is_submission_public": self.challenge_phase.is_submission_public,
+                "disable_logs": self.challenge_phase.disable_logs,
             }
         ]
         self.client.force_authenticate(user=None)
@@ -2573,6 +2576,7 @@ def test_get_challenge_phase_when_user_is_host(self):
                 "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
                 "is_submission_public": self.challenge_phase.is_submission_public,
+                "disable_logs": self.challenge_phase.disable_logs,
             },
             {
                 "id": self.private_challenge_phase.id,
@@ -2601,6 +2605,7 @@ def test_get_challenge_phase_when_user_is_host(self):
                 "default_submission_meta_attributes": self.private_challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.private_challenge_phase.allowed_email_ids,
                 "is_submission_public": self.private_challenge_phase.is_submission_public,
+                "disable_logs": self.private_challenge_phase.disable_logs,
             },
         ]
 
@@ -2951,6 +2956,7 @@ def test_get_particular_challenge_phase_if_user_is_participant(self):
             "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
             "allowed_email_ids": self.challenge_phase.allowed_email_ids,
             "is_submission_public": self.challenge_phase.is_submission_public,
+            "disable_logs": self.challenge_phase.disable_logs,
         }
         self.client.force_authenticate(user=self.participant_user)
         response = self.client.get(self.url, {})
@@ -2990,6 +2996,7 @@ def test_get_particular_challenge_phase_if_user_is_challenge_host(self):
             "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types,
             "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
             "allowed_email_ids": self.challenge_phase.allowed_email_ids,
+            "disable_logs": self.challenge_phase.disable_logs,
         }
         self.client.force_authenticate(user=self.user)
         response = self.client.get(self.url, {})
@@ -3052,6 +3059,7 @@ def test_update_challenge_phase_when_user_is_its_creator(self):
             "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
             "allowed_email_ids": self.challenge_phase.allowed_email_ids,
             "is_submission_public": self.challenge_phase.is_submission_public,
+            "disable_logs": self.challenge_phase.disable_logs,
         }
         response = self.client.put(
             self.url, {"name": new_name, "description": new_description}
@@ -3153,6 +3161,7 @@ def test_particular_challenge_phase_partial_update(self):
             "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
             "allowed_email_ids": self.challenge_phase.allowed_email_ids,
             "is_submission_public": self.challenge_phase.is_submission_public,
+            "disable_logs": self.challenge_phase.disable_logs,
         }
         response = self.client.patch(self.url, self.partial_update_data)
         self.assertEqual(response.data, expected)
@@ -4679,6 +4688,7 @@ def test_get_challenge_phase_by_pk(self):
             "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
             "allowed_email_ids": self.challenge_phase.allowed_email_ids,
             "is_submission_public": self.challenge_phase.is_submission_public,
+            "disable_logs": self.challenge_phase.disable_logs,
         }
         response = self.client.get(self.url, {})
         self.assertEqual(response.data, expected)
@@ -4754,6 +4764,7 @@ def test_get_challenge_phases_by_challenge_pk(self):
                 "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types,
                 "default_submission_meta_attributes": self.private_challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
+                "disable_logs": self.private_challenge_phase.disable_logs,
             },
             {
                 "id": self.challenge_phase.id,
@@ -4787,6 +4798,7 @@ def test_get_challenge_phases_by_challenge_pk(self):
                 "allowed_submission_file_types": self.challenge_phase.allowed_submission_file_types,
                 "default_submission_meta_attributes": self.challenge_phase.default_submission_meta_attributes,
                 "allowed_email_ids": self.challenge_phase.allowed_email_ids,
+                "disable_logs": self.challenge_phase.disable_logs,
             },
         ]
         response = self.client.get(self.url, {})