Merge pull request DIRACGrid#7331 from DIRACGridBot/cherry-pick-2-705…

…38bb7a-integration

[sweep:integration] fix: sets jobStatus=Failed/Payload failed iff the job was running

src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py

@@ -28,6 +28,7 @@
 from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient
 from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
 from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
+from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient
 from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
@@ -688,7 +689,7 @@ def _checkSubmittedJobs(self):
         payloadErrors = []
         originalJobID = self.jobReport.jobID
         for jobID, taskID in self.submissionDict.items():
-            if not taskID in self.computingElement.taskResults:
+            if taskID not in self.computingElement.taskResults:
                 continue
 
             result = self.computingElement.taskResults[taskID]
@@ -711,7 +712,12 @@ def _checkSubmittedJobs(self):
 
             # The payload failed (if result["Value"] is not 0)
             elif result["Value"]:
-                self.jobReport.setJobStatus(status=JobStatus.FAILED, minorStatus="Payload failed")
+                # In order to avoid overriding perfectly valid states, the status is updated iff the job was running
+                res = JobMonitoringClient().getJobsStatus(jobID)
+                if not res["OK"]:
+                    return res
+                if res["Value"][jobID]["Status"] == JobStatus.RUNNING:
+                    self.jobReport.setJobStatus(status=JobStatus.FAILED, minorStatus="Payload failed")
 
                 # Do not keep running and do not overwrite the Payload error
                 message = f"Payload execution failed with error code {result['Value']}"

src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py

@@ -3,6 +3,7 @@
 import os
 import pytest
 import time
+from unittest.mock import MagicMock
 
 from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Security.X509Chain import X509Chain  # pylint: disable=import-error
@@ -497,6 +498,7 @@ def test_submitAndCheckJob(mocker, localCE, job, expectedResult1, expectedResult
 
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__")
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobAgent.am_stopExecution")
+    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.JobMonitoringClient", return_value=MagicMock())
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.createJobWrapper", return_value=S_OK([jobName]))
     mocker.patch("DIRAC.Core.Security.X509Chain.X509Chain.dumpAllToString", return_value=S_OK())