From 8863cdcdb3feb49d51a24303cf10edca7d9e6b1a Mon Sep 17 00:00:00 2001 From: Stefano Belforte Date: Thu, 20 Feb 2025 13:05:48 +0100 Subject: [PATCH] remove "if useHtcV2". Always use V2. Fix #8935 (#8936) * remove "if useHtcV2". Always use V2. Fix #8935 * some pylint * a bit more pylint --- scripts/AdjustSites.py | 8 ++------ scripts/dag_bootstrap_startup.sh | 11 ----------- scripts/task_process/cache_status.py | 8 ++------ src/python/HTCondorLocator.py | 9 ++------- src/python/TaskWorker/Actions/DagmanCreator.py | 9 +++------ src/python/TaskWorker/Actions/DagmanKiller.py | 10 +++------- .../TaskWorker/Actions/DagmanResubmitter.py | 8 ++------ .../TaskWorker/Actions/DagmanSubmitter.py | 18 ++++-------------- src/python/TaskWorker/Actions/Handler.py | 6 ++---- src/python/TaskWorker/Actions/PostJob.py | 9 +++------ src/python/TaskWorker/Actions/PreDAG.py | 5 +---- src/python/TaskWorker/Actions/PreJob.py | 9 +++------ src/python/TaskWorker/Actions/RetryJob.py | 6 ++---- src/python/TaskWorker/Main.py | 10 +--------- src/python/TaskWorker/MasterWorker.py | 5 ----- 15 files changed, 30 insertions(+), 101 deletions(-) diff --git a/scripts/AdjustSites.py b/scripts/AdjustSites.py index 386fd5f3d4..8b16c7601b 100644 --- a/scripts/AdjustSites.py +++ b/scripts/AdjustSites.py @@ -31,12 +31,8 @@ from RESTInteractions import CRABRest from ServerUtilities import getProxiedWebDir, getColumn, downloadFromS3 -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad def printLog(msg): """ Utility function to print the timestamp in the log. Can be replaced diff --git a/scripts/dag_bootstrap_startup.sh b/scripts/dag_bootstrap_startup.sh index 8eaceaeeb7..bcbe8a53ef 100755 --- a/scripts/dag_bootstrap_startup.sh +++ b/scripts/dag_bootstrap_startup.sh @@ -33,11 +33,6 @@ fi srcname=$0 env > ${srcname%.sh}.env -if [ "X$useHtcV2" != "X" ]; then - echo "WILL USE HTC BINDINGS V2" # make a note in logfile dag_bootstrap.out - touch USE_HTC_V2_BINDINGS # create an empty file to make it easy to tell when investigating -fi - #Sourcing Remote Condor setup source_script=`grep '^RemoteCondorSetup =' $_CONDOR_JOB_AD | tr -d '"' | awk '{print $NF;}'` if [ "X$source_script" != "X" ] && [ -e $source_script ]; then @@ -251,12 +246,6 @@ Error = task_process/daemon.err.\$(Cluster).\$(Process) Queue 1 EOF - if [ "X$useHtcV2" != "X" ]; then - # add a directitve to the task_process submission JDL to pass this in the env. of the job - cat >> task_process/daemon.jdl << EOF -environment = "useHtcV2=True" -EOF - fi chmod +x task_process/task_proc_wrapper.sh condor_submit task_process/daemon.jdl else diff --git a/scripts/task_process/cache_status.py b/scripts/task_process/cache_status.py index 3d1eb52914..05b6f96b4d 100644 --- a/scripts/task_process/cache_status.py +++ b/scripts/task_process/cache_status.py @@ -14,12 +14,8 @@ import pickle import json -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad logging.basicConfig(filename='task_process/cache_status.log', level=logging.DEBUG) diff --git a/src/python/HTCondorLocator.py b/src/python/HTCondorLocator.py index d2eaae3f35..1e9642eb64 100644 --- a/src/python/HTCondorLocator.py +++ b/src/python/HTCondorLocator.py @@ -1,15 +1,10 @@ """ used by TaskWorker to decide which scheduler to submit to """ -import os import time import bisect import random -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad # From http://stackoverflow.com/questions/3679694/a-weighted-version-of-random-choice def weightedChoice(choices): diff --git a/src/python/TaskWorker/Actions/DagmanCreator.py b/src/python/TaskWorker/Actions/DagmanCreator.py index f6bb3829b6..490c5010e9 100644 --- a/src/python/TaskWorker/Actions/DagmanCreator.py +++ b/src/python/TaskWorker/Actions/DagmanCreator.py @@ -34,12 +34,9 @@ from WMCore.Services.CRIC.CRIC import CRIC from WMCore.WMRuntime.Tools.Scram import ARCH_TO_OS, SCRAM_TO_ARCH -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad + DAG_HEADER = """ diff --git a/src/python/TaskWorker/Actions/DagmanKiller.py b/src/python/TaskWorker/Actions/DagmanKiller.py index db033b76a9..df055dcf48 100644 --- a/src/python/TaskWorker/Actions/DagmanKiller.py +++ b/src/python/TaskWorker/Actions/DagmanKiller.py @@ -1,5 +1,4 @@ """ need a doc string here """ -import os import re from http.client import HTTPException from urllib.parse import urlencode @@ -11,12 +10,9 @@ from TaskWorker.Actions.TaskAction import TaskAction from TaskWorker.WorkerExceptions import TaskWorkerException -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad + WORKFLOW_RE = re.compile("[a-z0-9_]+") diff --git a/src/python/TaskWorker/Actions/DagmanResubmitter.py b/src/python/TaskWorker/Actions/DagmanResubmitter.py index b3de3386f2..388d4b35a8 100644 --- a/src/python/TaskWorker/Actions/DagmanResubmitter.py +++ b/src/python/TaskWorker/Actions/DagmanResubmitter.py @@ -13,12 +13,8 @@ from TaskWorker.WorkerExceptions import TaskWorkerException from TaskWorker.DataObjects import Result -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad class DagmanResubmitter(TaskAction): diff --git a/src/python/TaskWorker/Actions/DagmanSubmitter.py b/src/python/TaskWorker/Actions/DagmanSubmitter.py index 81cc564cc4..c5d66011ed 100644 --- a/src/python/TaskWorker/Actions/DagmanSubmitter.py +++ b/src/python/TaskWorker/Actions/DagmanSubmitter.py @@ -20,12 +20,8 @@ from TaskWorker.Actions import TaskAction from TaskWorker.WorkerExceptions import TaskWorkerException, SubmissionRefusedException -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad def addJobSubmitInfoToDagJobJDL(dagJdl, jobSubmit): @@ -500,8 +496,6 @@ def submitDirect(self, schedd, cmd, arg, dagJobJDL, task): environmentString += " CONDOR_ID=$(ClusterId).$(ProcId)" environmentString += " CRAB_RUNTIME_TARBALL=local CRAB_TASKMANAGER_TARBALL=local" ##SB environmentString += " " + " ".join(task['additional_environment_options'].split(';')) - if 'useHtcV2' in os.environ: - environmentString += " useHtcV2=True" # Environment command in JDL requires proper quotes https://htcondor.readthedocs.io/en/latest/man-pages/condor_submit.html#environment dagJobJDL["Environment"] = classad.quote(environmentString) dagJobJDL['My.CRAB_TaskLifetimeDays'] = str(TASKLIFETIME // 24 // 60 // 60) @@ -545,12 +539,8 @@ def submitDirect(self, schedd, cmd, arg, dagJobJDL, task): try: submitResult = schedd.submit(description=dagJobJDL, count=1, spool=True) clusterId = submitResult.cluster() - numProcs = submitResult.num_procs() - if 'useHtcV2' in os.environ: - schedd.spool(submitResult) - else: - myjobs = dagJobJDL.jobs(count=numProcs, clusterid=clusterId) - schedd.spool(list(myjobs)) + schedd.spool(submitResult) + except Exception as hte: raise TaskWorkerException(f"Submission failed with:\n{hte}") from hte diff --git a/src/python/TaskWorker/Actions/Handler.py b/src/python/TaskWorker/Actions/Handler.py index fe00054414..f0f5072663 100644 --- a/src/python/TaskWorker/Actions/Handler.py +++ b/src/python/TaskWorker/Actions/Handler.py @@ -29,10 +29,8 @@ from CRABUtils.TaskUtils import updateTaskStatus, uploadWarning from ServerUtilities import uploadToS3 -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor -else: - import htcondor +import htcondor2 as htcondor + class TaskHandler(): """Handling the set of operations to be performed.""" diff --git a/src/python/TaskWorker/Actions/PostJob.py b/src/python/TaskWorker/Actions/PostJob.py index 43bf7d5909..2c92a8bd90 100644 --- a/src/python/TaskWorker/Actions/PostJob.py +++ b/src/python/TaskWorker/Actions/PostJob.py @@ -112,12 +112,9 @@ from ServerUtilities import getLock, getHashLfn from RESTInteractions import CRABRest -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad + ASO_JOB = None G_JOB_REPORT_NAME = None diff --git a/src/python/TaskWorker/Actions/PreDAG.py b/src/python/TaskWorker/Actions/PreDAG.py index a3cba8e3af..53f1cf90cd 100644 --- a/src/python/TaskWorker/Actions/PreDAG.py +++ b/src/python/TaskWorker/Actions/PreDAG.py @@ -45,10 +45,7 @@ from TaskWorker.WorkerExceptions import TaskWorkerException from TaskWorker.Worker import failTask -if 'useHtcV2' in os.environ: - import classad2 as classad -else: - import classad +import classad2 as classad class PreDAG(): diff --git a/src/python/TaskWorker/Actions/PreJob.py b/src/python/TaskWorker/Actions/PreJob.py index 6f29dd3d47..db9b5b5c9f 100644 --- a/src/python/TaskWorker/Actions/PreJob.py +++ b/src/python/TaskWorker/Actions/PreJob.py @@ -16,12 +16,9 @@ from ServerUtilities import getWebdirForDb, insertJobIdSid from TaskWorker.Actions.RetryJob import JOB_RETURN_CODES -if 'useHtcV2' in os.environ: - import htcondor2 as htcondor - import classad2 as classad -else: - import htcondor - import classad +import htcondor2 as htcondor +import classad2 as classad + class PreJob: """ diff --git a/src/python/TaskWorker/Actions/RetryJob.py b/src/python/TaskWorker/Actions/RetryJob.py index 517e7af10e..d277482143 100644 --- a/src/python/TaskWorker/Actions/RetryJob.py +++ b/src/python/TaskWorker/Actions/RetryJob.py @@ -10,10 +10,8 @@ from ServerUtilities import executeCommand, getLock from ServerUtilities import MAX_DISK_SPACE, MAX_WALLTIME, MAX_MEMORY -if 'useHtcV2' in os.environ: - import classad2 as classad -else: - import classad +import classad2 as classad + JOB_RETURN_CODES = namedtuple('JobReturnCodes', 'OK RECOVERABLE_ERROR FATAL_ERROR')(0, 1, 2) diff --git a/src/python/TaskWorker/Main.py b/src/python/TaskWorker/Main.py index f6f0d1963e..f337dfca2f 100644 --- a/src/python/TaskWorker/Main.py +++ b/src/python/TaskWorker/Main.py @@ -8,6 +8,7 @@ import sys from WMCore.Configuration import loadConfigurationFile +from TaskWorker.MasterWorker import MasterWorker from TaskWorker.WorkerExceptions import ConfigException # NOTE: While importing the list of exceptions must be done before code starts @@ -76,15 +77,6 @@ def main(): if not status: raise ConfigException(msg) - # Only after loacConfiguratiob has set useHtcV2 in the environment - # we can importing HTCondorLocator and TaskWorker so that in all files we can use - # the env.vat. to decide if to import htcondor or htcondor2 - if getattr(configuration.TaskWorker, 'useHtcV2', None): - print("Configuration says to use HTC Bindings V2") - else: - print("Configuration says to use HTC Bindings V1") - from TaskWorker.MasterWorker import MasterWorker - if options.pdb: # override root loglevel to debug logging.getLogger().setLevel(logging.DEBUG) diff --git a/src/python/TaskWorker/MasterWorker.py b/src/python/TaskWorker/MasterWorker.py index dbd1c98cf4..2d78a84ee6 100644 --- a/src/python/TaskWorker/MasterWorker.py +++ b/src/python/TaskWorker/MasterWorker.py @@ -206,11 +206,6 @@ def logVersionAndConfig(config=None, logger=None): self.dbInstance = dbInstance self.logger.info('Will connect via URL: https://%s/%s', self.restHost, self.dbInstance) - if 'useHtcV2' in os.environ: - self.logger.info("Will use HTC python bindings V2") - else: - self.logger.info("Will use HTC python bindings V1") - #Let's increase the server's retries for recoverable errors in the MasterWorker #60 means we'll keep retrying for 1 hour basically (we retry at 20*NUMRETRY seconds, so at: 20s, 60s, 120s, 200s, 300s ...) self.crabserver = CRABRest(self.restHost, self.config.TaskWorker.cmscert, self.config.TaskWorker.cmskey, retry=20,