remove "if useHtcV2". Always use V2. Fix #8935 (#8936)

* remove "if useHtcV2". Always use V2. Fix #8935

* some pylint

* a bit more pylint

scripts/AdjustSites.py

@@ -31,12 +31,8 @@
 from RESTInteractions import CRABRest
 from ServerUtilities import getProxiedWebDir, getColumn, downloadFromS3
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
 
 def printLog(msg):
     """ Utility function to print the timestamp in the log. Can be replaced

scripts/dag_bootstrap_startup.sh

@@ -33,11 +33,6 @@ fi
 srcname=$0
 env > ${srcname%.sh}.env
 
-if [ "X$useHtcV2" != "X" ]; then
-  echo "WILL USE HTC BINDINGS V2"  # make a note in logfile dag_bootstrap.out
-  touch USE_HTC_V2_BINDINGS        # create an empty file to make it easy to tell when investigating
-fi
-
 #Sourcing Remote Condor setup
 source_script=`grep '^RemoteCondorSetup =' $_CONDOR_JOB_AD | tr -d '"' | awk '{print $NF;}'`
 if [ "X$source_script" != "X" ] && [ -e $source_script ]; then
@@ -251,12 +246,6 @@ Error         = task_process/daemon.err.\$(Cluster).\$(Process)
 Queue 1
 EOF
 
-    if [ "X$useHtcV2" != "X" ]; then
-      # add a directitve to the task_process submission JDL to pass this in the env. of the job
-      cat >> task_process/daemon.jdl << EOF
-environment = "useHtcV2=True"
-EOF
-    fi
         chmod +x task_process/task_proc_wrapper.sh
         condor_submit task_process/daemon.jdl
     else

scripts/task_process/cache_status.py

@@ -14,12 +14,8 @@
 import pickle
 import json
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
 
 
 logging.basicConfig(filename='task_process/cache_status.log', level=logging.DEBUG)

src/python/HTCondorLocator.py

@@ -1,15 +1,10 @@
 """ used by TaskWorker to decide which scheduler to submit to """
-import os
 import time
 import bisect
 import random
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
 
 # From http://stackoverflow.com/questions/3679694/a-weighted-version-of-random-choice
 def weightedChoice(choices):

src/python/TaskWorker/Actions/DagmanCreator.py

@@ -34,12 +34,9 @@
 from WMCore.Services.CRIC.CRIC import CRIC
 from WMCore.WMRuntime.Tools.Scram import ARCH_TO_OS, SCRAM_TO_ARCH
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
+
 
 DAG_HEADER = """
 

src/python/TaskWorker/Actions/DagmanKiller.py

@@ -1,5 +1,4 @@
 """ need a doc string here """
-import os
 import re
 from http.client import HTTPException
 from urllib.parse import urlencode
@@ -11,12 +10,9 @@
 from TaskWorker.Actions.TaskAction import TaskAction
 from TaskWorker.WorkerExceptions import TaskWorkerException
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
+
 
 WORKFLOW_RE = re.compile("[a-z0-9_]+")
 

src/python/TaskWorker/Actions/DagmanResubmitter.py

@@ -13,12 +13,8 @@
 from TaskWorker.WorkerExceptions import TaskWorkerException
 from TaskWorker.DataObjects import  Result
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
 
 
 class DagmanResubmitter(TaskAction):

src/python/TaskWorker/Actions/DagmanSubmitter.py

@@ -20,12 +20,8 @@
 from TaskWorker.Actions import TaskAction
 from TaskWorker.WorkerExceptions import TaskWorkerException, SubmissionRefusedException
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
 
 
 def addJobSubmitInfoToDagJobJDL(dagJdl, jobSubmit):
@@ -500,8 +496,6 @@ def submitDirect(self, schedd, cmd, arg, dagJobJDL, task):
         environmentString += " CONDOR_ID=$(ClusterId).$(ProcId)"
         environmentString += " CRAB_RUNTIME_TARBALL=local CRAB_TASKMANAGER_TARBALL=local"
         ##SB environmentString += " " + " ".join(task['additional_environment_options'].split(';'))
-        if 'useHtcV2' in os.environ:
-            environmentString += " useHtcV2=True"
         # Environment command in JDL requires proper quotes https://htcondor.readthedocs.io/en/latest/man-pages/condor_submit.html#environment
         dagJobJDL["Environment"] = classad.quote(environmentString)
         dagJobJDL['My.CRAB_TaskLifetimeDays'] = str(TASKLIFETIME // 24 // 60 // 60)
@@ -545,12 +539,8 @@ def submitDirect(self, schedd, cmd, arg, dagJobJDL, task):
         try:
             submitResult = schedd.submit(description=dagJobJDL, count=1, spool=True)
             clusterId = submitResult.cluster()
-            numProcs = submitResult.num_procs()
-            if 'useHtcV2' in os.environ:
-                schedd.spool(submitResult)
-            else:
-                myjobs = dagJobJDL.jobs(count=numProcs, clusterid=clusterId)
-                schedd.spool(list(myjobs))
+            schedd.spool(submitResult)
+
         except Exception as hte:
             raise TaskWorkerException(f"Submission failed with:\n{hte}") from hte
 

src/python/TaskWorker/Actions/Handler.py

@@ -29,10 +29,8 @@
 from CRABUtils.TaskUtils import updateTaskStatus, uploadWarning
 from ServerUtilities import uploadToS3
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-else:
-    import htcondor
+import htcondor2 as htcondor
+
 
 class TaskHandler():
     """Handling the set of operations to be performed."""

src/python/TaskWorker/Actions/PostJob.py

@@ -112,12 +112,9 @@
 from ServerUtilities import getLock, getHashLfn
 from RESTInteractions import CRABRest
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
+
 
 ASO_JOB = None
 G_JOB_REPORT_NAME = None

src/python/TaskWorker/Actions/PreDAG.py

@@ -45,10 +45,7 @@
 from TaskWorker.WorkerExceptions import TaskWorkerException
 from TaskWorker.Worker import failTask
 
-if 'useHtcV2' in os.environ:
-    import classad2 as classad
-else:
-    import classad
+import classad2 as classad
 
 
 class PreDAG():

src/python/TaskWorker/Actions/PreJob.py

@@ -16,12 +16,9 @@
 from ServerUtilities import getWebdirForDb, insertJobIdSid
 from TaskWorker.Actions.RetryJob import JOB_RETURN_CODES
 
-if 'useHtcV2' in os.environ:
-    import htcondor2 as htcondor
-    import classad2 as classad
-else:
-    import htcondor
-    import classad
+import htcondor2 as htcondor
+import classad2 as classad
+
 
 class PreJob:
     """

src/python/TaskWorker/Actions/RetryJob.py

@@ -10,10 +10,8 @@
 from ServerUtilities import executeCommand, getLock
 from ServerUtilities import MAX_DISK_SPACE, MAX_WALLTIME, MAX_MEMORY
 
-if 'useHtcV2' in os.environ:
-    import classad2 as classad
-else:
-    import classad
+import classad2 as classad
+
 
 JOB_RETURN_CODES = namedtuple('JobReturnCodes', 'OK RECOVERABLE_ERROR FATAL_ERROR')(0, 1, 2)
 

src/python/TaskWorker/Main.py

@@ -8,6 +8,7 @@
 import sys
 
 from WMCore.Configuration import loadConfigurationFile
+from TaskWorker.MasterWorker import MasterWorker
 from TaskWorker.WorkerExceptions import ConfigException
 
 # NOTE: While importing the list of exceptions must be done before code starts
@@ -76,15 +77,6 @@ def main():
     if not status:
         raise ConfigException(msg)
 
-    # Only after loacConfiguratiob has set useHtcV2 in the environment
-    # we can importing HTCondorLocator and TaskWorker so that in all files we can use
-    # the env.vat. to decide if to import htcondor or htcondor2
-    if getattr(configuration.TaskWorker, 'useHtcV2', None):
-        print("Configuration says to use HTC Bindings V2")
-    else:
-        print("Configuration says to use HTC Bindings V1")
-    from TaskWorker.MasterWorker import MasterWorker
-
     if options.pdb:
         # override root loglevel to debug
         logging.getLogger().setLevel(logging.DEBUG)

src/python/TaskWorker/MasterWorker.py

@@ -206,11 +206,6 @@ def logVersionAndConfig(config=None, logger=None):
         self.dbInstance = dbInstance
         self.logger.info('Will connect via URL: https://%s/%s', self.restHost, self.dbInstance)
 
-        if 'useHtcV2' in os.environ:
-            self.logger.info("Will use HTC python bindings V2")
-        else:
-            self.logger.info("Will use HTC python bindings V1")
-
         #Let's increase the server's retries for recoverable errors in the MasterWorker
         #60 means we'll keep retrying for 1 hour basically (we retry at 20*NUMRETRY seconds, so at: 20s, 60s, 120s, 200s, 300s ...)
         self.crabserver = CRABRest(self.restHost, self.config.TaskWorker.cmscert, self.config.TaskWorker.cmskey, retry=20,