From 18ac4c7db4b7658ed0397abf88473960fc376e24 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Oct 2023 17:13:20 +0200 Subject: [PATCH 1/4] fix: always binding the WN to the VO, if specified --- src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 6554ceb540a..4f7fbbafc99 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -1031,6 +1031,9 @@ def _getPilotOptions(self, queue, **kwargs): pilotOptions.append(f"-Q {self.queueDict[queue]['QueueName']}") # SiteName pilotOptions.append(f"-n {queueDict['Site']}") + # VO + if self.vo: + pilotOptions.append(f"--wnVO={self.vo}") # Generic Options if "GenericOptions" in queueDict: From 3e1865167363de1efda7aac9a8fc9765b3823a26 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Oct 2023 17:13:58 +0200 Subject: [PATCH 2/4] docs: minor docs adjustments --- docs/source/AdministratorGuide/Systems/Framework/index.rst | 2 +- src/DIRAC/Core/scripts/install_full.cfg | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/AdministratorGuide/Systems/Framework/index.rst b/docs/source/AdministratorGuide/Systems/Framework/index.rst index 5a83fef0f06..e51159de3a3 100644 --- a/docs/source/AdministratorGuide/Systems/Framework/index.rst +++ b/docs/source/AdministratorGuide/Systems/Framework/index.rst @@ -15,7 +15,7 @@ and a monitoring system that accounts for CPU and memory usage, queries served, Another very important functionality provided by the framework system is proxies management, via the ProxyManager service and database. -ComponentMonitoring, SecurityLogging, and ProxyManager services are only part of the services that constitute the +ComponentMonitoring and ProxyManager services are only part of the services that constitute the Framework of DIRAC. The following sections add some details for some of the Framework systems. diff --git a/src/DIRAC/Core/scripts/install_full.cfg b/src/DIRAC/Core/scripts/install_full.cfg index fd96990c860..d8a6f6f2f0c 100755 --- a/src/DIRAC/Core/scripts/install_full.cfg +++ b/src/DIRAC/Core/scripts/install_full.cfg @@ -95,8 +95,6 @@ LocalInstallation Services += DataManagement/TornadoFTS3Manager Services += DataManagement/TornadoS3Gateway Services += Framework/BundleDelivery - Services += Framework/SecurityLogging - Services += Framework/TornadoNotification Services += Framework/TornadoProxyManager Services += Framework/TornadoTokenManager Services += Framework/TornadoUserProfileManager From dac681462db5688a5986f21fed1627eac3145e6f Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 3 Oct 2023 17:14:13 +0200 Subject: [PATCH 3/4] style: import ordering --- .../Client/Helpers/Operations.py | 7 ++++--- src/DIRAC/Core/Utilities/Shifter.py | 7 +++---- .../WorkloadManagementSystem/Client/Matcher.py | 16 +++++++--------- .../Service/MatcherHandler.py | 6 ++---- .../private/ConfigHelper.py | 4 ++-- .../scripts/dirac_admin_add_pilot.py | 3 ++- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py index 8e06d7c2113..5796ee25278 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py @@ -33,12 +33,13 @@ but this works iff the object is instantiated by a proxy (and not, e.g., using a server certificate) """ -import os import _thread + from diraccfg import CFG -from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.ConfigurationSystem.Client.Helpers import Registry, CSGlobals + +from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.Core.Utilities import LockRing from DIRAC.Core.Utilities.DErrno import ESECTION diff --git a/src/DIRAC/Core/Utilities/Shifter.py b/src/DIRAC/Core/Utilities/Shifter.py index 98d57b968b0..775956ad02e 100644 --- a/src/DIRAC/Core/Utilities/Shifter.py +++ b/src/DIRAC/Core/Utilities/Shifter.py @@ -3,12 +3,11 @@ """ import os -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Registry, cfgPath +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities.File import mkDir from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath -from DIRAC.ConfigurationSystem.Client.Helpers import Registry def getShifterProxy(shifterType, fileName=False): diff --git a/src/DIRAC/WorkloadManagementSystem/Client/Matcher.py b/src/DIRAC/WorkloadManagementSystem/Client/Matcher.py index 118d0681b24..359658f068c 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/Matcher.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/Matcher.py @@ -4,20 +4,18 @@ """ import time -from DIRAC import gLogger, convertToPy3VersionNumber - -from DIRAC.Core.Utilities.PrettyPrint import printDict -from DIRAC.Core.Security import Properties +from DIRAC import convertToPy3VersionNumber, gLogger from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.WorkloadManagementSystem.Client import JobStatus +from DIRAC.Core.Security import Properties +from DIRAC.Core.Utilities.PrettyPrint import printDict +from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus +from DIRAC.WorkloadManagementSystem.Client import JobStatus, PilotStatus from DIRAC.WorkloadManagementSystem.Client.Limiter import Limiter -from DIRAC.WorkloadManagementSystem.Client import PilotStatus -from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, singleValueDefFields, multiValueMatchFields -from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB -from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus +from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB +from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, multiValueMatchFields, singleValueDefFields class PilotVersionError(Exception): diff --git a/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py index cf9f65d5f66..d346511a29b 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py @@ -3,16 +3,14 @@ It uses a Matcher and a Limiter object that encapsulates the matching logic. It connects to JobDB, TaskQueueDB, JobLoggingDB, and PilotAgentsDB. """ -from DIRAC import S_OK, S_ERROR - +from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader - -from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher, PilotVersionError from DIRAC.WorkloadManagementSystem.Client.Limiter import Limiter +from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher, PilotVersionError class MatcherHandlerMixin: diff --git a/src/DIRAC/WorkloadManagementSystem/private/ConfigHelper.py b/src/DIRAC/WorkloadManagementSystem/private/ConfigHelper.py index 6c7d117ffca..a2f23945f83 100644 --- a/src/DIRAC/WorkloadManagementSystem/private/ConfigHelper.py +++ b/src/DIRAC/WorkloadManagementSystem/private/ConfigHelper.py @@ -1,8 +1,8 @@ """ A set of utilities for getting configuration information for the WMS components """ -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Operations, Registry from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers import Registry, Operations def findGenericPilotCredentials(vo=False, group=False, pilotDN="", pilotGroup=""): diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py index 72ba14a3c17..f1187ba088c 100644 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py @@ -12,7 +12,8 @@ """ -from DIRAC import S_OK, S_ERROR, gLogger, exit as DIRACExit +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC import exit as DIRACExit from DIRAC.Core.Base.Script import Script From e6d15221926856f6cff041eb56e7c940665ed8e7 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 4 Oct 2023 16:01:39 +0200 Subject: [PATCH 4/4] test: always adding 1 job for DIRAC.Jenkins.ch --- src/DIRAC/tests/Utilities/testJobDefinitions.py | 16 ++++++++++++++++ tests/System/unitTestUserJobs.py | 6 +++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/tests/Utilities/testJobDefinitions.py b/src/DIRAC/tests/Utilities/testJobDefinitions.py index 3e7e44e0492..cf6fe309076 100644 --- a/src/DIRAC/tests/Utilities/testJobDefinitions.py +++ b/src/DIRAC/tests/Utilities/testJobDefinitions.py @@ -65,6 +65,22 @@ def helloWorld(): return endOfAllJobs(J) +def helloWorldJenkins(): + """simple hello world job to Jenkins""" + + J = baseToAllJobs("helloWorldJenkins") + try: + J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]]) + except IndexError: + try: + J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]]) + except IndexError: # we are in Jenkins + J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]]) + J.setExecutable("exe-script.py", "", "helloWorld.log") + J.setDestination("DIRAC.Jenkins.ch") + return endOfAllJobs(J) + + def helloWorldCERN(): """simple hello world job to CERN""" diff --git a/tests/System/unitTestUserJobs.py b/tests/System/unitTestUserJobs.py index 137ac708c41..c874ea85137 100644 --- a/tests/System/unitTestUserJobs.py +++ b/tests/System/unitTestUserJobs.py @@ -1,8 +1,8 @@ """ Collection of user jobs for testing purposes """ # pylint: disable=wrong-import-position, invalid-name -import unittest import time +import unittest import DIRAC @@ -37,6 +37,10 @@ def test_submit(self): self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"]) + res = helloWorldJenkins() + self.assertTrue(res["OK"]) + jobsSubmittedList.append(res["Value"]) + res = helloWorldCERN() self.assertTrue(res["OK"]) jobsSubmittedList.append(res["Value"])