Merge pull request #7226 from fstagni/81_fixes15

[8.1] SiteDirector adds wnVO Pilot options, if needed

docs/source/AdministratorGuide/Systems/Framework/index.rst

@@ -15,7 +15,7 @@ and a monitoring system that accounts for CPU and memory usage, queries served,
 Another very important functionality provided by the framework system is proxies management,
 via the ProxyManager service and database.
 
-ComponentMonitoring, SecurityLogging, and ProxyManager services are only part of the services that constitute the
+ComponentMonitoring and ProxyManager services are only part of the services that constitute the
 Framework of DIRAC.
 
 The following sections add some details for some of the Framework systems.

src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py

@@ -33,12 +33,13 @@
     but this works iff the object is instantiated by a proxy (and not, e.g., using a server certificate)
 
 """
-import os
 import _thread
+
 from diraccfg import CFG
-from DIRAC import S_OK, S_ERROR, gConfig
-from DIRAC.ConfigurationSystem.Client.Helpers import Registry, CSGlobals
+
+from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
+from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry
 from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.Core.Utilities import LockRing
 from DIRAC.Core.Utilities.DErrno import ESECTION

src/DIRAC/Core/Utilities/Shifter.py

@@ -3,12 +3,11 @@
 """
 import os
 
-from DIRAC import S_OK, S_ERROR, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
+from DIRAC.ConfigurationSystem.Client.Helpers import Registry, cfgPath
+from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Utilities.File import mkDir
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
-from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
-from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 
 
 def getShifterProxy(shifterType, fileName=False):

src/DIRAC/Core/scripts/install_full.cfg

@@ -95,8 +95,6 @@ LocalInstallation
   Services += DataManagement/TornadoFTS3Manager
   Services += DataManagement/TornadoS3Gateway
   Services += Framework/BundleDelivery
-  Services += Framework/SecurityLogging
-  Services += Framework/TornadoNotification
   Services += Framework/TornadoProxyManager
   Services += Framework/TornadoTokenManager
   Services += Framework/TornadoUserProfileManager

src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1031,6 +1031,9 @@ def _getPilotOptions(self, queue, **kwargs):
         pilotOptions.append(f"-Q {self.queueDict[queue]['QueueName']}")
         # SiteName
         pilotOptions.append(f"-n {queueDict['Site']}")
+        # VO
+        if self.vo:
+            pilotOptions.append(f"--wnVO={self.vo}")
 
         # Generic Options
         if "GenericOptions" in queueDict:

src/DIRAC/WorkloadManagementSystem/Client/Matcher.py

@@ -4,20 +4,18 @@
 """
 import time
 
-from DIRAC import gLogger, convertToPy3VersionNumber
-
-from DIRAC.Core.Utilities.PrettyPrint import printDict
-from DIRAC.Core.Security import Properties
+from DIRAC import convertToPy3VersionNumber, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
-from DIRAC.WorkloadManagementSystem.Client import JobStatus
+from DIRAC.Core.Security import Properties
+from DIRAC.Core.Utilities.PrettyPrint import printDict
+from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
+from DIRAC.WorkloadManagementSystem.Client import JobStatus, PilotStatus
 from DIRAC.WorkloadManagementSystem.Client.Limiter import Limiter
-from DIRAC.WorkloadManagementSystem.Client import PilotStatus
-from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, singleValueDefFields, multiValueMatchFields
-from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
-from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus
+from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
+from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, multiValueMatchFields, singleValueDefFields
 
 
 class PilotVersionError(Exception):

src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py

@@ -3,16 +3,14 @@
     It uses a Matcher and a Limiter object that encapsulates the matching logic.
     It connects to JobDB, TaskQueueDB, JobLoggingDB, and PilotAgentsDB.
 """
-from DIRAC import S_OK, S_ERROR
-
+from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.DISET.RequestHandler import RequestHandler
 from DIRAC.Core.Utilities import DErrno
 from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
-
-from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher, PilotVersionError
 from DIRAC.WorkloadManagementSystem.Client.Limiter import Limiter
+from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher, PilotVersionError
 
 
 class MatcherHandlerMixin:

src/DIRAC/WorkloadManagementSystem/private/ConfigHelper.py

@@ -1,8 +1,8 @@
 """ A set of utilities for getting configuration information for the WMS components
 """
-from DIRAC import S_OK, S_ERROR, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
+from DIRAC.ConfigurationSystem.Client.Helpers import Operations, Registry
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
-from DIRAC.ConfigurationSystem.Client.Helpers import Registry, Operations
 
 
 def findGenericPilotCredentials(vo=False, group=False, pilotDN="", pilotGroup=""):

src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_add_pilot.py

@@ -12,7 +12,8 @@
 
 """
 
-from DIRAC import S_OK, S_ERROR, gLogger, exit as DIRACExit
+from DIRAC import S_ERROR, S_OK, gLogger
+from DIRAC import exit as DIRACExit
 from DIRAC.Core.Base.Script import Script
 
 

src/DIRAC/tests/Utilities/testJobDefinitions.py

@@ -65,6 +65,22 @@ def helloWorld():
     return endOfAllJobs(J)
 
 
+def helloWorldJenkins():
+    """simple hello world job to Jenkins"""
+
+    J = baseToAllJobs("helloWorldJenkins")
+    try:
+        J.setInputSandbox([find_all("exe-script.py", rootPath, "DIRAC/tests/Workflow")[0]])
+    except IndexError:
+        try:
+            J.setInputSandbox([find_all("exe-script.py", ".", "DIRAC/tests/Workflow")[0]])
+        except IndexError:  # we are in Jenkins
+            J.setInputSandbox([find_all("exe-script.py", os.environ["WORKSPACE"], "DIRAC/tests/Workflow")[0]])
+    J.setExecutable("exe-script.py", "", "helloWorld.log")
+    J.setDestination("DIRAC.Jenkins.ch")
+    return endOfAllJobs(J)
+
+
 def helloWorldCERN():
     """simple hello world job to CERN"""
 

tests/System/unitTestUserJobs.py

@@ -1,8 +1,8 @@
 """ Collection of user jobs for testing purposes
 """
 # pylint: disable=wrong-import-position, invalid-name
-import unittest
 import time
+import unittest
 
 import DIRAC
 
@@ -37,6 +37,10 @@ def test_submit(self):
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])
 
+        res = helloWorldJenkins()
+        self.assertTrue(res["OK"])
+        jobsSubmittedList.append(res["Value"])
+
         res = helloWorldCERN()
         self.assertTrue(res["OK"])
         jobsSubmittedList.append(res["Value"])