Fix8815 (#8961)

* remove latest cache key * Using CI_PIPELINE_ID to prevent race conditions * add CI_PIPELINE_ID
dmwm · Feb 28, 2025 · 4bca823 · 4bca823
1 parent c51853e
commit 4bca823
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 62 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -270,21 +270,9 @@ task_submission:
   cache:
     - key: $CI_PIPELINE_ID
       paths:
-        - workdir/submitted_tasks_CV
-        - workdir/submitted_tasks_CCV
-        - workdir/submitted_tasks_TS
-      policy: push
-    - key: submitted_tasks_TS_latest
-      paths:
-        - workdir/submitted_tasks_TS
-      policy: push
-    - key: submitted_tasks_CV_latest
-      paths:
-        - workdir/submitted_tasks_CV
-      policy: push
-    - key: submitted_tasks_CCV_latest
-      paths:
-        - workdir/submitted_tasks_CCV
+        - workdir/submitted_tasks_CV_${CI_PIPELINE_ID}
+        - workdir/submitted_tasks_CCV_${CI_PIPELINE_ID}
+        - workdir/submitted_tasks_TS_${CI_PIPELINE_ID}
       policy: push
 
 client_validation_suite:
@@ -307,16 +295,12 @@ client_validation_suite:
     - export ROOT_DIR
     - export CMSSW_release=CMSSW_13_0_2
     - export Client_Validation_Suite=true
-    - ls workdir/submitted_tasks_CV
+    - ls workdir/submitted_tasks_CV_${CI_PIPELINE_ID}
     - bash -x cicd/gitlab/CV_config.sh
   cache:
     - key: $CI_PIPELINE_ID
       paths:
-        - workdir/submitted_tasks_CV
-      policy: pull
-    - key: submitted_tasks_CV_latest
-      paths:
-        - workdir/submitted_tasks_CV
+        - workdir/submitted_tasks_CV_${CI_PIPELINE_ID}
       policy: pull
 
 client_configuration_validation:
@@ -339,16 +323,12 @@ client_configuration_validation:
     - export ROOT_DIR
     - export CMSSW_release=CMSSW_13_0_2
     - export Client_Configuration_Validation=true
-    - ls workdir/submitted_tasks_CCV
+    - ls workdir/submitted_tasks_CCV_${CI_PIPELINE_ID}
     - bash cicd/gitlab/retry.sh bash -x cicd/gitlab/CCV_config.sh
   cache:
     - key: $CI_PIPELINE_ID
       paths:
-        - workdir/submitted_tasks_CCV
-      policy: pull
-    - key: submitted_tasks_CCV_latest
-      paths:
-        - workdir/submitted_tasks_CCV
+        - workdir/submitted_tasks_CCV_${CI_PIPELINE_ID}
       policy: pull
 
 check_status_tracking:
@@ -374,11 +354,7 @@ check_status_tracking:
   cache:
     - key: $CI_PIPELINE_ID
       paths:
-        - workdir/submitted_tasks_TS
-      policy: pull
-    - key: submitted_tasks_TS_latest
-      paths:
-        - workdir/submitted_tasks_TS
+        - workdir/submitted_tasks_TS_${CI_PIPELINE_ID}
       policy: pull
 
 # if test is pass, retag with `*-stable`

diff --git a/cicd/gitlab/CCV_config.sh b/cicd/gitlab/CCV_config.sh
@@ -58,15 +58,15 @@ TEST_RESULT='FAILED'
 MESSAGE='Test failed. Investigate manually'
 
 # Define an associative array to hold the test results
-declare -A results=( ["SUCCESSFUL"]="successful_tests" ["FAILED"]="failed_tests" ["RETRY"]="retry_tests" )
+declare -A results=( ["SUCCESSFUL"]="successful_tests_${CI_PIPELINE_ID}" ["FAILED"]="failed_tests_${CI_PIPELINE_ID}" ["RETRY"]="retry_tests_${CI_PIPELINE_ID}" )
 
 for result in "${!results[@]}"; do
 	if [ -s "${results[$result]}" ]; then
 		test_result=$(cat "${results[$result]}")
 		echo -e "\n${result} TESTS:\n${test_result}" >> message_CCVResult_interim
 
 		# Handle retry logic
-		if [ "${results[$result]}" == "retry_tests" ]; then
+		if [ "${results[$result]}" == "retry_tests_${CI_PIPELINE_ID}" ]; then
             TEST_RESULT='FULL-STATUS-UNKNOWN'
             if [ "$RETRY" -ge "$MAX_RETRY" ]; then
                 MESSAGE='Exceeded configured retries. If needed restart manually.'
@@ -80,11 +80,11 @@ for result in "${!results[@]}"; do
 done
 
 # Check if the tests passed or failed
-if [ -s "failed_tests" ]; then
+if [ -s "failed_tests_${CI_PIPELINE_ID}" ]; then
     TEST_RESULT='FAILED'
-elif [ -s "retry_tests" ]; then
+elif [ -s "retry_tests_${CI_PIPELINE_ID}" ]; then
     TEST_RESULT='FULL-STATUS-UNKNOWN'
-elif [ -s "successful_tests" ]; then
+elif [ -s "successful_tests_${CI_PIPELINE_ID}" ]; then
     TEST_RESULT='SUCCEEDED'
     MESSAGE='Test is done.'
 fi

diff --git a/cicd/gitlab/clientConfigurationValidation.sh b/cicd/gitlab/clientConfigurationValidation.sh
@@ -19,13 +19,13 @@ source "${ROOT_DIR}/cicd/gitlab/setupCRABClient.sh"
 python ${ROOT_DIR}/test/makeTests.py
 
 # Ensure the log files exist (creating successful_tests (if doesn't exist) and deleting/recreating retry_tests and failed_tests)
-touch ${WORK_DIR}/successful_tests
+touch ${WORK_DIR}/successful_tests_${CI_PIPELINE_ID}
 
 # Delete and recreate retry_tests and failed_tests (if they exist)
-rm -f ${WORK_DIR}/retry_tests ${WORK_DIR}/failed_tests
-touch ${WORK_DIR}/retry_tests ${WORK_DIR}/failed_tests
+rm -f ${WORK_DIR}/retry_tests ${WORK_DIR}/failed_tests_${CI_PIPELINE_ID}
+touch ${WORK_DIR}/retry_tests ${WORK_DIR}/failed_tests_${CI_PIPELINE_ID}
 
-if [ -f "${WORK_DIR}/submitted_tasks_CCV" ]; then
+if [ -f "${WORK_DIR}/submitted_tasks_CCV_${CI_PIPELINE_ID}" ]; then
   while read task; do
     echo "Processing task: $task"
 
@@ -38,15 +38,15 @@ if [ -f "${WORK_DIR}/submitted_tasks_CCV" ]; then
     echo "Exit Code: $retVal"
 
     if [ $retVal -eq 0 ]; then
-      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/successful_tests
+      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/successful_tests_${CI_PIPELINE_ID}
     elif [ $retVal -eq 2 ]; then
-      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/retry_tests
+      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/retry_tests_${CI_PIPELINE_ID}
     else
-      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/failed_tests
+      echo ${test_to_execute}-check.sh ${task} - $retVal >> ${WORK_DIR}/failed_tests_${CI_PIPELINE_ID}
     fi
-  done <"${WORK_DIR}/submitted_tasks_CCV"
+  done <"${WORK_DIR}/submitted_tasks_CCV_${CI_PIPELINE_ID}"
 else
-  echo "Error: ${WORK_DIR}/submitted_tasks_CCV is not a file"
+  echo "Error: ${WORK_DIR}/submitted_tasks_CCV_${CI_PIPELINE_ID} is not a file"
   exit 1
 fi
 
diff --git a/cicd/gitlab/clientValidation.sh b/cicd/gitlab/clientValidation.sh
@@ -184,7 +184,7 @@ source "${ROOT_DIR}/cicd/gitlab/setupCRABClient.sh"
   # START CHECKING SUBMITTED TASK'S STATUS
   ##################################################
 
-  TASKTOTRACK=`cat ${WORK_DIR}/submitted_tasks_CV`
+  TASKTOTRACK=`cat ${WORK_DIR}/submitted_tasks_CV_${CI_PIPELINE_ID}`
   PROJDIR=`crab remake --task=$TASKTOTRACK --instance=$REST_Instance --proxy=$PROXY| grep 'Finished remaking project directory' | awk '{print $6}'`
 
   ### 5. test crab preparelocal --proxy=PROXY --dir=PROJDIR

diff --git a/cicd/gitlab/executeTests.sh b/cicd/gitlab/executeTests.sh
@@ -16,6 +16,7 @@ echo "(DEBUG) Client_Validation_Suite: ${Client_Validation_Suite}"
 echo "(DEBUG) Client_Configuration_Validation: ${Client_Configuration_Validation}"
 echo "(DEBUG) Task_Submission_Status_Tracking: ${Task_Submission_Status_Tracking}"
 echo "(DEBUG) Check_Publication_Status: ${Check_Publication_Status}"
+echo "(DEBUG) CI_PIPELINE_ID: ${CI_PIPELINE_ID}"
 
 # always run inside ./workdir
 export ROOT_DIR=$PWD
@@ -49,7 +50,7 @@ if $ERR ; then
     echo -e "Something went wrong during task submission. None of the downstream jobs were triggered."
     exit 1
 else
-    declare -A tests=( ["Task_Submission_Status_Tracking"]=submitted_tasks_TS ["Client_Validation_Suite"]=submitted_tasks_CV ["Client_Configuration_Validation"]=submitted_tasks_CCV)
+    declare -A tests=( ["Task_Submission_Status_Tracking"]=submitted_tasks_TS_${CI_PIPELINE_ID} ["Client_Validation_Suite"]=submitted_tasks_CV_${CI_PIPELINE_ID} ["Client_Configuration_Validation"]=submitted_tasks_CCV_${CI_PIPELINE_ID})
     for test in "${!tests[@]}";
     do
         path="${WORK_DIR}/${tests[$test]}"

diff --git a/cicd/gitlab/st/statusTracking.py b/cicd/gitlab/st/statusTracking.py
@@ -142,7 +142,7 @@ def main():
     # Read all tasks from the specified files into a single list
     tasks = [
     line 
-    for file_name in ['submitted_tasks_TS']
+    for file_name in [f"submitted_tasks_TS_{os.environ['CI_PIPELINE_ID']}"]
     if os.path.exists(f'{work_dir}/{file_name}')
     for line in open(f'{work_dir}/{file_name}').readlines()
     ]

diff --git a/cicd/gitlab/taskSubmission.sh b/cicd/gitlab/taskSubmission.sh
@@ -33,11 +33,11 @@ submitTasks() {
       submitExitCode=$?
       echo ${output}
       if [ $submitExitCode != 0 ]; then
-          echo ${file_name} >> "${WORK_DIR}"/failed_tasks
+          echo ${file_name} >> "${WORK_DIR}"/failed_tasks_${CI_PIPELINE_ID}
           killAfterFailure
       else
-          echo ${output} | grep -oP '(?<=Task name:\s)[^\s]*(?=)' >> "${WORK_DIR}"/submitted_tasks_${2}
-          echo ${output} | grep -oP '(?<=Project dir:\s)[^\s]*(?=)' >> "${WORK_DIR}"/project_directories
+          echo ${output} | grep -oP '(?<=Task name:\s)[^\s]*(?=)' >> "${WORK_DIR}"/submitted_tasks_${2}_${CI_PIPELINE_ID}
+          echo ${output} | grep -oP '(?<=Project dir:\s)[^\s]*(?=)' >> "${WORK_DIR}"/project_directories_${CI_PIPELINE_ID}
       fi
   done
 }
@@ -46,35 +46,35 @@ immediateCheck(){
 #Run immediate check on tasks submitted for Client_Configuration_Validation testing
 #Save results to failed_CCV_tests and successful_CCV_tests files
 
- project_dir="/tmp/crabTestConfig"
+ project_dir="/tmp/crabTestConfig_${CI_PIPELINE_ID}"
  mkdir -p "${project_dir}"  # Ensure project_dir exists
- touch "${WORK_DIR}/successful_CCV_tests"
- touch "${WORK_DIR}/failed_CCV_tests"
+ touch "${WORK_DIR}/successful_CCV_tests_${CI_PIPELINE_ID}"
+ touch "${WORK_DIR}/failed_CCV_tests_${CI_PIPELINE_ID}"
  for task in ${tasksToCheck};
  do
      echo -e "\nRunning immediate test on task: ${task}"
      test_to_execute=`echo "${task}" | grep -oP '(?<=_crab_).*(?=)'`
      task_dir=${project_dir}/crab_${test_to_execute}
      bash -x ${test_to_execute}-testSubmit.sh ${task_dir} && \
-       echo ${test_to_execute}-testSubmit.sh ${task_dir} - $? >> ${WORK_DIR}/successful_CCV_tests || \
-       echo ${test_to_execute}-testSubmit.sh ${task_dir} - $? >> ${WORK_DIR}/failed_CCV_tests
+       echo ${test_to_execute}-testSubmit.sh ${task_dir} - $? >> ${WORK_DIR}/successful_CCV_tests_${CI_PIPELINE_ID} || \
+       echo ${test_to_execute}-testSubmit.sh ${task_dir} - $? >> ${WORK_DIR}/failed_CCV_tests_${CI_PIPELINE_ID}
  done
 }
 
 killAfterFailure(){
 #In case at least one task submission failed, kill all succesfully submitted tasks
- if [ -e "${WORK_DIR}/project_directories" ]; then
+ if [ -e "${WORK_DIR}/project_directories_${CI_PIPELINE_ID}" ]; then
    echo -e "\nTask submission failed. Killing submitted tasks.\n"
    while read task ; do
        echo "Killing task: ${task}"
        crab kill --dir=${task} --proxy=${PROXY}
        killExitCode=$?
        if [ $killExitCode != 0 ]; then
-           echo -e "Failed to kill: ${task}" >> ${WORK_DIR}/killed_tasks
+           echo -e "Failed to kill: ${task}" >> ${WORK_DIR}/killed_tasks_${CI_PIPELINE_ID}
        else
-           echo -e "Successfully killed: ${task}" >> ${WORK_DIR}/killed_tasks
+           echo -e "Successfully killed: ${task}" >> ${WORK_DIR}/killed_tasks_${CI_PIPELINE_ID}
        fi
-   done <${WORK_DIR}/project_directories
+   done <${WORK_DIR}/project_directories_${CI_PIPELINE_ID}
  fi
  exit 1
 }
@@ -94,7 +94,7 @@ if [ "${Client_Configuration_Validation}" = true ]; then
     python ${ROOT_DIR}/test/makeTests.py
     filesToSubmit=`find . -maxdepth 1 -type f -name '*.py' ! -name '*PSET*'`
     submitTasks "${filesToSubmit}" "CCV"
-    tasksToCheck=`cat ${WORK_DIR}/submitted_tasks_CCV`
+    tasksToCheck=`cat ${WORK_DIR}/submitted_tasks_CCV_${CI_PIPELINE_ID}`
     immediateCheck "${tasksToCheck}"
     cd ${WORK_DIR}
 fi