-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #275 from pneerincx/master
Updated templates for local and slurm backends and rewrote tests to produce output in a test specific subdir.
- Loading branch information
Showing
329 changed files
with
15,101 additions
and
11,619 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
25 changes: 19 additions & 6 deletions
25
molgenis-compute-core/src/main/resources/templates/localhost/footer.ftl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,22 @@ | ||
| <#noparse># | ||
| ## Start of footer for backend 'local'. | ||
| # | ||
| ## General footer | ||
| # | ||
|
|
||
| # Show that we successfully finished. If the .finished file exists, then this step will be skipped when you resubmit your workflow | ||
| touch $ENVIRONMENT_DIR/${taskId}.sh.finished | ||
| if [ -d "${MC_tmpFolder:-}" ]; then | ||
| echo -n "INFO: Removing MC_tmpFolder ${MC_tmpFolder} ..." | ||
| rm -rf "${MC_tmpFolder}" | ||
| echo 'done.' | ||
| fi | ||
|
|
||
| tS=${SECONDS:-0} | ||
| tM=$((SECONDS / 60 )) | ||
| tH=$((SECONDS / 3600)) | ||
| echo "On $(date +"%Y-%m-%d %T") ${MC_jobScript} finished successfully after ${tM} minutes." >> molgenis.bookkeeping.log | ||
| printf '%s:\t%d seconds\t%d minutes\t%d hours\n' "${MC_jobScript}" "${tS}" "${tM}" "${tH}" >> molgenis.bookkeeping.walltime | ||
|
|
||
| mv "${MC_jobScript}".{started,finished} | ||
|
|
||
| trap - EXIT | ||
| exit 0 | ||
|
|
||
| # Also do bookkeeping | ||
| echo "On $(date +"%Y-%m-%d %T"), after $(( ($(date +%s) - $MOLGENIS_START) / 60 )) minutes, task ${taskId}.sh finished successfully" >> $ENVIRONMENT_DIR/molgenis.bookkeeping.log | ||
| </#noparse> |
9 changes: 0 additions & 9 deletions
9
molgenis-compute-core/src/main/resources/templates/localhost/footer_gaf.ftl
This file was deleted.
Oops, something went wrong.
182 changes: 100 additions & 82 deletions
182
molgenis-compute-core/src/main/resources/templates/localhost/header.ftl
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,101 +1,119 @@ | ||
| #!/bin/bash | ||
| # | ||
| ## Header for 'local' backend | ||
| # This is job <#if project??>${project}_</#if>${taskId} | ||
| # | ||
|
|
||
| #highly recommended to use | ||
| #set -e # exit if any subcommand or pipeline returns a non-zero status | ||
| #set -u # exit if any uninitialised variable is used | ||
|
|
||
| # Set location of *.env and *.log files | ||
| ENVIRONMENT_DIR="." | ||
| # | ||
| ## Start of header for backend 'local'. | ||
| # | ||
|
|
||
| # If you detect an error, then exit your script by calling this function | ||
| exitWithError(){ | ||
| errorCode=$1 | ||
| errorMessage=$2 | ||
| echo "$errorCode: $errorMessage --- TASK '${taskId}.sh' --- ON $(date +"%Y-%m-%d %T"), AFTER RUNNING $(( ($(date +%s) - $MOLGENIS_START) / 60 )) MINUTES" >> $ENVIRONMENT_DIR/molgenis.error.log | ||
| exit $errorCode | ||
| } | ||
| set -e | ||
| set -u | ||
|
|
||
| # For bookkeeping how long your task takes | ||
| MOLGENIS_START=$(date +%s) | ||
| ENVIRONMENT_DIR='.' | ||
|
|
||
| # Show that the task has started | ||
| touch $ENVIRONMENT_DIR/${taskId}.sh.started | ||
| # | ||
| # Variables declared in MOLGENIS Compute headers/footers always start with an MC_ prefix. | ||
| # | ||
| declare MC_jobScript="${taskId}.sh" | ||
| declare MC_jobScriptSTDERR="${taskId}.err" | ||
| declare MC_jobScriptSTDOUT="${taskId}.out" | ||
| declare MC_failedFile="molgenis.pipeline.failed" | ||
|
|
||
| <#noparse> | ||
| getFile() | ||
| { | ||
| ARGS=($@) | ||
| NUMBER="${#ARGS[@]}"; | ||
| if [ "$NUMBER" -eq "1" ] | ||
| then | ||
| myFile=${ARGS[0]} | ||
|
|
||
| if test ! -e $myFile; | ||
| then | ||
| echo "WARNING in getFile/putFile: $myFile is missing" 1>&2 | ||
| fi | ||
| declare MC_singleSeperatorLine=$(head -c 120 /dev/zero | tr '\0' '-') | ||
| declare MC_doubleSeperatorLine=$(head -c 120 /dev/zero | tr '\0' '=') | ||
| declare MC_tmpFolder='tmpFolder' | ||
| declare MC_tmpFile='tmpFile' | ||
| declare MC_tmpFolderCreated=0 | ||
|
|
||
| else | ||
| echo "Example usage: getData \"\$TMPDIR/datadir/myfile.txt\"" | ||
| fi | ||
| } | ||
| # | ||
| ## | ||
| ### Header functions. | ||
| ## | ||
| # | ||
|
|
||
| putFile() | ||
| { | ||
| `getFile $@` | ||
| function errorExitAndCleanUp() { | ||
| local _signal="${1}" | ||
| local _problematicLine="${2}" | ||
| local _exitStatus="${3:-$?}" | ||
| local _executionHost="$(hostname)" | ||
| local _format='INFO: Last 50 lines or less of %s:\n' | ||
| local _errorMessage="FATAL: Trapped ${_signal} signal in ${MC_jobScript} running on ${_executionHost}. Exit status code was ${_exitStatus}." | ||
| if [ "${_signal}" == 'ERR' ]; then | ||
| _errorMessage="FATAL: Trapped ${_signal} signal on line ${_problematicLine} in ${MC_jobScript} running on ${_executionHost}. Exit status code was ${_exitStatus}." | ||
| fi | ||
| _errorMessage=${4:-"${_errorMessage}"} # Optionally use custom error message as 4th argument. | ||
| echo "${_errorMessage}" | ||
| echo "${MC_doubleSeperatorLine}" > "${MC_failedFile}" | ||
| echo "${_errorMessage}" >> "${MC_failedFile}" | ||
| if [ -f "${MC_jobScriptSTDERR}" ]; then | ||
| echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}" | ||
| printf "${_format}" "${MC_jobScriptSTDERR}" >> "${MC_failedFile}" | ||
| echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}" | ||
| tail -50 "${MC_jobScriptSTDERR}" >> "${MC_failedFile}" | ||
| fi | ||
| if [ -f "${MC_jobScriptSTDOUT}" ]; then | ||
| echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}" | ||
| printf "${_format}" "${MC_jobScriptSTDOUT}" >> "${MC_failedFile}" | ||
| echo "${MC_singleSeperatorLine}" >> "${MC_failedFile}" | ||
| tail -50 "${MC_jobScriptSTDOUT}" >> "${MC_failedFile}" | ||
| fi | ||
| echo "${MC_doubleSeperatorLine}" >> "${MC_failedFile}" | ||
| } | ||
|
|
||
| inputs() | ||
| { | ||
| for name in $@ | ||
| do | ||
| if test ! -e $name; | ||
| then | ||
| echo "$name is missing" 1>&2 | ||
| exit 1; | ||
| fi | ||
| done | ||
| # | ||
| # Create tmp dir per script/job. | ||
| # To be called with with either a file or folder as first and only argument. | ||
| # Defines two globally set variables: | ||
| # 1. MC_tmpFolder: a tmp dir for this job/script. When function is called multiple times MC_tmpFolder will always be the same. | ||
| # 2. MC_tmpFile: when the first argument was a folder, MC_tmpFile == MC_tmpFolder | ||
| # when the first argument was a file, MC_tmpFile will be a path to a tmp file inside MC_tmpFolder. | ||
| # | ||
| function makeTmpDir { | ||
| # | ||
| # Compile paths. | ||
| # | ||
| local _originalPath="${1}" | ||
| local _myMD5="$(md5sum ${MC_jobScript} | cut -d ' ' -f 1)" | ||
| local _tmpSubFolder="tmp_${MC_jobScript}_${_myMD5}" | ||
| local _dir | ||
| local _base | ||
| if [[ -d "${_originalPath}" ]]; then | ||
| _dir="${_originalPath}" | ||
| _base='' | ||
| else | ||
| _base=$(basename "${_originalPath}") | ||
| _dir=$(dirname "${_originalPath}") | ||
| fi | ||
| MC_tmpFolder="${_dir}/${_tmpSubFolder}/" | ||
| MC_tmpFile="${MC_tmpFolder}/${_base}" | ||
| echo "DEBUG ${MC_jobScript}::makeTmpDir: dir='${_dir}';base='${_base}';MC_tmpFile='${MC_tmpFile}'" | ||
| # | ||
| # Cleanup the previously created tmpFolder first if this script was resubmitted. | ||
| # | ||
| if [[ ${MC_tmpFolderCreated} -eq 0 && -d "${MC_tmpFolder}" ]]; then | ||
| rm -rf "${MC_tmpFolder}" | ||
| fi | ||
| # | ||
| # (Re-)create tmpFolder. | ||
| # | ||
| mkdir -p "${MC_tmpFolder}" | ||
| MC_tmpFolderCreated=1 | ||
| } | ||
|
|
||
| outputs() | ||
| { | ||
| for name in $@ | ||
| do | ||
| if test -e $name; | ||
| then | ||
| echo "skipped" | ||
| echo "skipped" 1>&2 | ||
| exit 0; | ||
| else | ||
| return; | ||
| fi | ||
| done | ||
| } | ||
| trap 'errorExitAndCleanUp HUP NA $?' HUP | ||
| trap 'errorExitAndCleanUp INT NA $?' INT | ||
| trap 'errorExitAndCleanUp QUIT NA $?' QUIT | ||
| trap 'errorExitAndCleanUp TERM NA $?' TERM | ||
| trap 'errorExitAndCleanUp EXIT NA $?' EXIT | ||
| trap 'errorExitAndCleanUp ERR $LINENO $?' ERR | ||
|
|
||
| alloutputsexist() | ||
| { | ||
| all_exist=true | ||
| for name in $@ | ||
| do | ||
| if test ! -e $name; | ||
| then | ||
| all_exist=false | ||
| fi | ||
| done | ||
| if $all_exist; | ||
| then | ||
| echo "skipped" | ||
| echo "skipped" 1>&2 | ||
| sleep 30 | ||
| exit 0; | ||
| else | ||
| return 0; | ||
| fi | ||
| } | ||
| </#noparse> | ||
| touch "${MC_jobScript}.started" | ||
|
|
||
| # | ||
| ## End of header for 'local' backend | ||
| ## End of header for backend 'local' | ||
| # | ||
|
|
||
| </#noparse> |
43 changes: 0 additions & 43 deletions
43
molgenis-compute-core/src/main/resources/templates/localhost/header_gaf.ftl
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.