From 66e38518c49a1ebc39d7e02010d503ac2170e302 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 6 Nov 2024 16:20:45 +0100 Subject: [PATCH] sweep: #7873 added some tests jobs for certifications --- .../tests/Utilities/testJobDefinitions.py | 64 ++++++++++++++- .../exe-script-with-input-single-location.py | 9 +++ tests/System/exe-script-with-input.py | 9 +++ tests/System/testInputFile.txt | 1 + tests/System/testInputFileSingleLocation.txt | 3 + tests/System/transformationSystem.sh | 2 +- tests/System/transformation_replication.sh | 2 +- tests/System/unitTestUserJobs.py | 77 +++++++++---------- 8 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 tests/System/exe-script-with-input-single-location.py create mode 100644 tests/System/exe-script-with-input.py create mode 100644 tests/System/testInputFile.txt create mode 100644 tests/System/testInputFileSingleLocation.txt diff --git a/src/DIRAC/tests/Utilities/testJobDefinitions.py b/src/DIRAC/tests/Utilities/testJobDefinitions.py index c3b73ea999d..b6e79786a0d 100644 --- a/src/DIRAC/tests/Utilities/testJobDefinitions.py +++ b/src/DIRAC/tests/Utilities/testJobDefinitions.py @@ -3,14 +3,13 @@ # pylint: disable=invalid-name +import errno import os import time -import errno from DIRAC import rootPath -from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac - +from DIRAC.Interfaces.API.Job import Job from DIRAC.tests.Utilities.utils import find_all @@ -78,6 +77,43 @@ def helloWorldJenkins(): J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]]) J.setExecutable("exe-script.py", "", "helloWorld.log") J.setDestination("DIRAC.Jenkins.ch") + + +def helloWorld_input(): + """simple hello world job with input""" + + J = baseToAllJobs("helloWorld_input") + try: + J.setInputSandbox([find_all("exe-script-with-input.py", rootPath, "DIRAC/tests/Workflow")[0]]) + except IndexError: + try: + J.setInputSandbox([find_all("exe-script-with-input.py", ".", "DIRAC/tests/Workflow")[0]]) + except IndexError: # we are in Jenkins + J.setInputSandbox( + [find_all("exe-script-with-input.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]] + ) + J.setExecutable("exe-script-with-input.py", "", "helloWorld.log") + return endOfAllJobs(J) + + +def helloWorld_input_single(): + """simple hello world job with input""" + + J = baseToAllJobs("helloWorld_input_single") + try: + J.setInputSandbox([find_all("exe-script-with-input-single-location.py", rootPath, "DIRAC/tests/Workflow")[0]]) + except IndexError: + try: + J.setInputSandbox([find_all("exe-script-with-input-single-location.py", ".", "DIRAC/tests/Workflow")[0]]) + except IndexError: # we are in Jenkins + J.setInputSandbox( + [ + find_all( + "exe-script-with-input-single-location.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow" + )[0] + ] + ) + J.setExecutable("exe-script-with-input-single-location.py", "", "helloWorld.log") return endOfAllJobs(J) @@ -326,6 +362,28 @@ def parametricJob(): return endOfAllJobs(J) +def parametricJobInputData(): + """Creates a parametric job with 3 subjobs which are simple hello world jobs, but with input data""" + + J = baseToAllJobs("parametricJobInput") + try: + J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]]) + except IndexError: + try: + J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]]) + except IndexError: # we are in Jenkins + J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]]) + J.setParameterSequence("args", ["one", "two", "three"]) + J.setParameterSequence("iargs", [1, 2, 3]) + J.setParameterSequence( + "InputData", + ["/dteam/user/f/fstagni/test/1.txt", "/dteam/user/f/fstagni/test/2.txt", "/dteam/user/f/fstagni/test/3.txt"], + ) + J.setInputDataPolicy("download") + J.setExecutable("exe-script.py", arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log") + return endOfAllJobs(J) + + def jobWithOutput(): """Creates a job that uploads an output. The output SE is not set here, so it would use the default /Resources/StorageElementGroups/SE-USER diff --git a/tests/System/exe-script-with-input-single-location.py b/tests/System/exe-script-with-input-single-location.py new file mode 100644 index 00000000000..dd3d250f3be --- /dev/null +++ b/tests/System/exe-script-with-input-single-location.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +"""Script to run Executable application""" +from os import system +import sys + +# Main +if __name__ == "__main__": + sys.exit(int(system("""cat testInputFileSingleLocation.txt""") / 256)) diff --git a/tests/System/exe-script-with-input.py b/tests/System/exe-script-with-input.py new file mode 100644 index 00000000000..2c779e74b29 --- /dev/null +++ b/tests/System/exe-script-with-input.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python + +"""Script to run Executable application""" +import sys +from os import system + +# Main +if __name__ == "__main__": + sys.exit(int(system("""cat testInputFile.txt""") / 256)) diff --git a/tests/System/testInputFile.txt b/tests/System/testInputFile.txt new file mode 100644 index 00000000000..70556506c57 --- /dev/null +++ b/tests/System/testInputFile.txt @@ -0,0 +1 @@ +this is just a test diff --git a/tests/System/testInputFileSingleLocation.txt b/tests/System/testInputFileSingleLocation.txt new file mode 100644 index 00000000000..3900ab252f9 --- /dev/null +++ b/tests/System/testInputFileSingleLocation.txt @@ -0,0 +1,3 @@ +this is just a test + +This file should be at /dteam/user/f/fstagni/test/testInputFileSingleLocation.txt in RAL-SE diff --git a/tests/System/transformationSystem.sh b/tests/System/transformationSystem.sh index c3419586404..50494236b23 100755 --- a/tests/System/transformationSystem.sh +++ b/tests/System/transformationSystem.sh @@ -45,7 +45,7 @@ directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime #selecting a random USER Storage Element #SEs=$(dirac-dms-show-se-status |grep USER |grep -v 'Banned\|Degraded\|-2' | awk '{print $1}') #get all SEs ending with -SE that are Active -SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|CESNET-SE\|Banned\|Probing\|Error\|-new' | awk '{print $1}') +SEs=$(dirac-dms-show-se-status | grep -e "-SE \|-disk" | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}') x=0 for n in $SEs diff --git a/tests/System/transformation_replication.sh b/tests/System/transformation_replication.sh index d4943d0fa53..4195a6fb17d 100755 --- a/tests/System/transformation_replication.sh +++ b/tests/System/transformation_replication.sh @@ -45,7 +45,7 @@ echo "Creating TransformationSystemTest" mkdir -p TransformationSystemTest directory=/dteam/diracCertification/Test/INIT/$version/$tdate/$stime/replication #get all SEs ending with -SE that are Active, randomize, turn into array -SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|CESNET\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) ) +SEs=( $(dirac-dms-show-se-status | grep -e "-SE \|-disk " | grep -v 'RAL\|Banned\|Probing\|Error\|-new' | awk '{print $1}' | sort -R | xargs) ) SOURCE_SE=${SEs[0]} TARGET_SE=${SEs[1]} diff --git a/tests/System/unitTestUserJobs.py b/tests/System/unitTestUserJobs.py index 8e904ae4ac2..82c33bcef4d 100644 --- a/tests/System/unitTestUserJobs.py +++ b/tests/System/unitTestUserJobs.py @@ -1,6 +1,7 @@ """ Collection of user jobs for testing purposes """ # pylint: disable=wrong-import-position, invalid-name +import sys import time import unittest @@ -22,7 +23,29 @@ class GridSubmissionTestCase(unittest.TestCase): """Base class for the Regression test cases""" def setUp(self): - pass + result = getProxyInfo() + if result["Value"]["group"] not in ["dteam_user", "gridpp_user"]: + print("GET A USER GROUP") + sys.exit(1) + + res = DataManager().getReplicas( + [ + "/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt", + "/dteam/user/f/fstagni/test/testInputFile.txt", + ] + ) + if not res["OK"]: + print(f"DATAMANAGER.getRepicas failure: {res['Message']}") + sys.exit(1) + if res["Value"]["Failed"]: + print(f"DATAMANAGER.getRepicas failed for something: {res['Value']['Failed']}") + sys.exit(1) + + replicas = res["Value"]["Successful"] + if list(replicas["/dteam/user/f/fstagni/test/testInputFile.txt"]) != ["RAL-SE", "UKI-LT2-IC-HEP-disk"]: + print("/dteam/user/f/fstagni/test/testInputFile.txt locations are not correct") + if list(replicas["/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt"]) != ["RAL-SE"]: + print("/dteam/user/f/fstagni/test/testInputFileSingleLocation.txt locations are not correct") def tearDown(self): pass @@ -41,6 +64,14 @@ def test_submit(self): self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"]) + res = helloWorld_input() + self.assertTrue(res["OK"]) + jobsSubmittedList.append(res["Value"]) + + res = helloWorld_input_single() + self.assertTrue(res["OK"]) + jobsSubmittedList.append(res["Value"]) + res = helloWorldCERN() self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"]) @@ -93,6 +124,10 @@ def test_submit(self): self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"]) + res = parametricJobInputData() + self.assertTrue(res["OK"]) + jobsSubmittedList.append(res["Value"]) + res = jobWithOutput() self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"]) @@ -104,45 +139,6 @@ def test_submit(self): print(f"submitted {len(jobsSubmittedList)} jobs: {','.join(str(js) for js in jobsSubmittedList)}") -# FIXME: This is also in the extension...? To try! -# class monitorSuccess( GridSubmissionTestCase ): -# -# def test_monitor( self ): -# -# toRemove = [] -# fail = False -# -# # we will check every 10 minutes, up to 6 hours -# counter = 0 -# while counter < 36: -# jobStatus = self.dirac.status( jobsSubmittedList ) -# self.assertTrue( jobStatus['OK'] ) -# for jobID in jobsSubmittedList: -# status = jobStatus['Value'][jobID]['Status'] -# minorStatus = jobStatus['Value'][jobID]['MinorStatus'] -# if status == 'Done': -# self.assertTrue( minorStatus in ['Execution Complete', 'Requests Done'] ) -# jobsSubmittedList.remove( jobID ) -# res = self.dirac.getJobOutputLFNs( jobID ) -# if res['OK']: -# lfns = res['Value'] -# toRemove += lfns -# if status in ['Failed', 'Killed', 'Deleted']: -# fail = True -# jobsSubmittedList.remove( jobID ) -# if jobsSubmittedList: -# time.sleep( 600 ) -# counter = counter + 1 -# else: -# break -# -# # removing produced files -# res = self.dirac.removeFile( toRemove ) -# self.assertTrue( res['OK'] ) -# -# if fail: -# self.assertFalse( True ) - ############################################################################# # Test Suite run ############################################################################# @@ -150,5 +146,4 @@ def test_submit(self): if __name__ == "__main__": suite = unittest.defaultTestLoader.loadTestsFromTestCase(GridSubmissionTestCase) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(submitSuccess)) - # suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( monitorSuccess ) ) testResult = unittest.TextTestRunner(verbosity=2).run(suite)