From a65231e3691c919b4e9a1995d08cd0ba41420fca Mon Sep 17 00:00:00 2001 From: bpuchala Date: Wed, 28 Feb 2018 10:13:57 -0500 Subject: [PATCH 1/5] update conda build script --- build_conda.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/build_conda.sh b/build_conda.sh index 5d4dfec..b53a556 100644 --- a/build_conda.sh +++ b/build_conda.sh @@ -9,13 +9,17 @@ if [ -z $LOCATION ]; then fi anaconda upload --user prisms-center $LOCATION -# if on linux, convert to osx -conda convert --platform osx-64 $LOCATION -o conda-recipes -anaconda upload --user prisms-center conda-recipes/osx-64/prisms-jobs* +VERSION=`echo $LOCATION | perl -ne '/prisms-jobs-(.*)-/ && print $1'` -# if on osx, convert to linux -conda convert --platform linux-64 $LOCATION -o conda-recipes -anaconda upload --user prisms-center conda-recipes/linux-64/prisms-jobs* +if [[ "$OSTYPE" == "linux-gnu" ]]; then + # if on linux, convert to osx + conda convert --platform osx-64 $LOCATION -o conda-recipes + anaconda upload --user prisms-center conda-recipes/osx-64/prisms-jobs-$VERSION* +elif [[ "$OSTYPE" == "darwin"* ]]; then + # if on osx, convert to linux + conda convert --platform linux-64 $LOCATION -o conda-recipes + anaconda upload --user prisms-center conda-recipes/linux-64/prisms-jobs-$VERSION* +fi # finish anaconda logout From 316cd3e34563260f7a33b8b9dbac3d0b5c3d3eb3 Mon Sep 17 00:00:00 2001 From: bpuchala Date: Fri, 15 Feb 2019 12:03:34 -0500 Subject: [PATCH 2/5] update build_conda.sh --- build_conda.sh | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/build_conda.sh b/build_conda.sh index b53a556..a6d6378 100644 --- a/build_conda.sh +++ b/build_conda.sh @@ -1,25 +1,32 @@ -# begin -anaconda login +# function to allow passing conda build command line arguments +build_conda () { + # build, get location of result, upload + conda build $1 conda-recipes/prisms_jobs > conda-recipes/tmp.out + LOCATION=$(grep 'conda upload' conda-recipes/tmp.out | cut -f3 -d ' ') + if [ -z $LOCATION ]; then + LOCATION=$(grep 'Nothing to test for' conda-recipes/tmp.out | cut -f5 -d ' ') + fi + anaconda upload --user prisms-center $LOCATION -# build, get location of result, upload -conda build conda-recipes/prisms_jobs > conda-recipes/tmp.out -LOCATION=$(grep 'conda upload' conda-recipes/tmp.out | cut -f3 -d ' ') -if [ -z $LOCATION ]; then - LOCATION=$(grep 'Nothing to test for' conda-recipes/tmp.out | cut -f5 -d ' ') -fi -anaconda upload --user prisms-center $LOCATION + if [[ "$OSTYPE" == "linux-gnu" ]]; then + OTHER=osx-64 + elif [[ "$OSTYPE" == "darwin"* ]]; then + OTHER=linux-64 + fi + + # if on linux, convert to osx + conda convert --platform $OTHER $LOCATION -o conda-recipes + + FILE=$(basename $LOCATION) + anaconda upload --user prisms-center conda-recipes/$OTHER/$FILE + +} -VERSION=`echo $LOCATION | perl -ne '/prisms-jobs-(.*)-/ && print $1'` +# begin +anaconda login -if [[ "$OSTYPE" == "linux-gnu" ]]; then - # if on linux, convert to osx - conda convert --platform osx-64 $LOCATION -o conda-recipes - anaconda upload --user prisms-center conda-recipes/osx-64/prisms-jobs-$VERSION* -elif [[ "$OSTYPE" == "darwin"* ]]; then - # if on osx, convert to linux - conda convert --platform linux-64 $LOCATION -o conda-recipes - anaconda upload --user prisms-center conda-recipes/linux-64/prisms-jobs-$VERSION* -fi +build_conda "--python 2.7" +build_conda "--python 3.6" # finish anaconda logout From dfcb588d3d74101487832ed1503a0695d3964f52 Mon Sep 17 00:00:00 2001 From: bpuchala Date: Tue, 26 Feb 2019 09:09:07 -0500 Subject: [PATCH 3/5] fix 'write_submit_script' issues --- prisms_jobs/config.py | 58 ++++++++++++++++++----------- prisms_jobs/interface/slurm.py | 60 +++++++++++++++--------------- prisms_jobs/interface/torque.py | 66 ++++++++++++++++----------------- 3 files changed, 100 insertions(+), 84 deletions(-) diff --git a/prisms_jobs/config.py b/prisms_jobs/config.py index 0544109..83ac3a7 100644 --- a/prisms_jobs/config.py +++ b/prisms_jobs/config.py @@ -14,10 +14,11 @@ __settings = None __software = None +__write_submit_script = None __update_selection_method = None _IMPORT_WARNING_MSG = """\ -prisms_jobs does not detect any job management software +prisms_jobs does not detect any job management software and the 'PRISMS_JOBS_SOFTWARE' environment variable is not set. """ @@ -27,7 +28,7 @@ def detect_software(): Returns: A string naming the job management software, or None. Possibilities are: - + * 'torque' - detected via 'qsub' * 'slurm' - detected via 'sbatch' """ @@ -44,17 +45,17 @@ def set_software(software_name=None): Args: software_name (str, optional, default=None): - + =================== ======================================================= 'torque' TORQUE 'slurm' SLURM The name of an existing findable python module None or 'default' Empty stub, does nothing =================== ======================================================= - + Raises: prisms_jobs.JobDBError: If software_name is unrecognized - + """ if software_name is None: software_name = 'default' @@ -91,7 +92,7 @@ def _default_update_selection(curs): def _check_hostname_update_selection(curs): """Select jobs with jobstatus!='C' and matching hostname""" hostname = socket.gethostname() - + # Parse our hostname so we can only select jobs from THIS host # Otherwise, if we're on a multiple-clusters-same-home setup, # we may incorrectly update jobs from one cluster onto the other @@ -107,15 +108,15 @@ def _check_hostname_update_selection(curs): def set_update_selection_method(update_method=None): """Enable customization of which jobs are selected for JobDB.update() - + Args: update_method (str, optional): - + =================== ======================================================= 'default' (or None) Select jobs with jobstatus != 'C' 'check_hostname' Select jobs with jobstatus != 'C' and matching hostname =================== ======================================================= - + Raises: prisms_jobs.JobsError: For unexpected value. """ @@ -135,6 +136,19 @@ def update_selection_method(): configure() return __update_selection_method +def set_write_submit_script(write_submit_script=None): + """If true, write submit script to file and then submit job; else submit via command line.""" + global __write_submit_script + if __write_submit_script is None: + __write_submit_script = write_submit_script + return __write_submit_script + +def write_submit_script(): + """If true, write submit script to file and then submit job; else submit via command line.""" + if __write_submit_script is None: + configure() + return __write_submit_script + def config_dir(): """Return configuration directory""" @@ -148,13 +162,13 @@ def config_path(dir=None): def default_settings(dir=None): """Default configuration dictionary - + Args: dir (str): Location of the directory storing the config.json file. - + Notes: See configure for details. - + Returns: Dict with configuration settings. """ @@ -169,15 +183,15 @@ def default_settings(dir=None): def read_config(dir=None): """Read configuration file. - + Note: Will create with default values if not existing. - + Args: dir (str, optional): Location of the directory storing the config.json file. The default location is ``$PRISMS_JOBS_DIR/.prisms_jobs``, where the environment variable ``PRISMS_JOBS_DIR``=``$HOME`` if not set. - + Returns: Dict with configuration settings. """ @@ -205,14 +219,14 @@ def read_config(dir=None): def write_config(dir=None, settings=None): """Write current configuration settings to file. - + Args: dir (str, optional): Location of the directory storing the config.json file. The default location is ``$PRISMS_JOBS_DIR/.prisms_jobs``, where the environment variable ``PRISMS_JOBS_DIR``=``$HOME`` if not set. settings (dict, optional): Settings to write to config.json file. Uses current settings by default. - + Returns: Dict with configuration settings. """ @@ -238,9 +252,9 @@ def dbpath(): def configure(settings=None): """Set configuration - + Sets the global configuration settings dictionary. Options are: - + * 'dbpath': (str) Location of the SQLite jobs database. * 'software': (str) @@ -251,11 +265,11 @@ def configure(settings=None): * 'update_method': (str, default='default') Controls which jobs are updated when JobDB.update() is called. See set_update_selection_method for options. - + The values are then used to update: * software: Module used to interface with job submission software * update_selection_method: Function used by JobDB.update() - + Args: config_dict (dict, optional): All configuration settings. Default reads configuration settings using read_config. @@ -265,5 +279,5 @@ def configure(settings=None): global __settings __settings = settings set_software(__settings['software']) + set_write_submit_script(__settings['write_submit_script']) set_update_selection_method(__settings['update_method']) - diff --git a/prisms_jobs/interface/slurm.py b/prisms_jobs/interface/slurm.py index 4a76a1d..737a973 100644 --- a/prisms_jobs/interface/slurm.py +++ b/prisms_jobs/interface/slurm.py @@ -14,6 +14,7 @@ from io import StringIO ### Internal ### +import prisms_jobs from prisms_jobs import JobsError from prisms_jobs.misc import getlogin, run, seconds @@ -92,7 +93,7 @@ def _squeue(jobid=None, username=getlogin(), full=False, sformat=None): #pyli def sub_string(job): """Write Job as a string suitable for slurm - + Args: prisms_jobs.Job: Job to be submitted """ @@ -122,24 +123,26 @@ def sub_string(job): # jobstr += "#SBATCH -N {0}\n".format(job.nodes) if job.queue is not None: jobstr += "#SBATCH -p {0}\n".format(job.queue) + if job.constraint is not None: + jobstr += "#SBATCH --constraint={0}\n".format(job.constraint) jobstr += "{0}\n".format(job.command) return jobstr def job_id(all=False, name=None): #pylint: disable=redefined-builtin """Get job IDs - + Args: - all (bool): If True, use ``squeue`` to query all user jobs. Else, check + all (bool): If True, use ``squeue`` to query all user jobs. Else, check ``SLURM_JOBID`` environment variable for ID of current job. - + name (str): If all==True, use name to filter results. - + Returns: One of str, List(str), or None: - Returns a str if all==False and ``SLURM_JOBID`` exists, a List(str) + Returns a str if all==False and ``SLURM_JOBID`` exists, a List(str) if all==True, else None. - + """ if all or name is not None: jobid = [] @@ -163,7 +166,7 @@ def job_rundir(jobid): Args: jobid (str or List(str)): IDs of jobs to get the run directory - + Returns: dict: A dict, with id:rundir pairs. @@ -189,11 +192,11 @@ def job_status(jobid=None): IDs of jobs to query for status. None for all user jobs. Returns: - + dict of dict: - + The outer dict uses jobid as key; the inner dict contains: - + ================ ====================================================== "name" Job name "nodes" Number of nodes @@ -319,14 +322,14 @@ def submit(substr, write_submit_script=None): Args: substr (str): The submit script string - write_submit_script (bool, optional): If true, submit via file skipping + write_submit_script (bool, optional): If true, submit via file skipping lines containing '#SBATCH -J'; otherwise, submit via commandline. If not specified, uses ``prisms_jobs.config['write_submit_script']``. - - + + Returns: str: ID of submitted job - + Raises: JobsError: If a submission error occurs """ @@ -338,10 +341,10 @@ def submit(substr, write_submit_script=None): raise JobsError( None, r"""Error in pbs.misc.submit(). Jobname ("#SBATCH\s+-J\s+(.*)\s") not found in submit string.""") - + if write_submit_script is None: - write_submit_script = prisms_jobs.config['write_submit_script'] - + write_submit_script = prisms_jobs.config.write_submit_script() + if write_submit_script: if os.path.exists(jobname): index = 0 @@ -366,37 +369,37 @@ def submit(substr, write_submit_script=None): def delete(jobid): """``scancel`` a job. - + Args: jobid (str): ID of job to cancel - + Returns: int: ``scancel`` returncode - + """ return run(["scancel", jobid])[2] def hold(jobid): """``scontrol`` delay a job. - + Args: jobid (str): ID of job to delay (for 30days) - + Returns: int: ``scontrol`` returncode - + """ return run(["scontrol", "update", "JobId=", jobid, "StartTime=", "now+30days"])[2] def release(jobid): """``scontrol`` un-delay a job. - + Args: jobid (str): ID of job to release - + Returns: int: ``scontrol`` returncode - + """ return run(["scontrol", "update", "JobId=", jobid, "StartTime=", "now"])[2] @@ -406,7 +409,7 @@ def alter(jobid, arg): Args: jobid (str): ID of job to alter arg (str): 'arg' is a scontrol command option string. For instance, "-a 201403152300.19" - + Returns: int: ``scontrol`` returncode """ @@ -415,4 +418,3 @@ def alter(jobid, arg): def read(job, qsubstr): """Raise exception""" raise Exception("primsms_jobs.read is not yet implemented for Slurm") - diff --git a/prisms_jobs/interface/torque.py b/prisms_jobs/interface/torque.py index d396764..7c0f804 100644 --- a/prisms_jobs/interface/torque.py +++ b/prisms_jobs/interface/torque.py @@ -13,6 +13,7 @@ from io import StringIO from six import iteritems, string_types +import prisms_jobs from prisms_jobs import JobsError from prisms_jobs.misc import getlogin, run, seconds @@ -27,7 +28,7 @@ def _getversion(): # call 'qstat' using subprocess stdout = run(opt)[0] - + # return the version number return stdout.rstrip("\n").lower().lstrip("version: ") @@ -97,7 +98,7 @@ def _qstat(jobid=None, username=getlogin(), full=False): def sub_string(job): """Write Job as a string suitable for torque - + Args: job (prisms_jobs.Job instance): Job to be submitted """ @@ -131,18 +132,18 @@ def sub_string(job): def job_id(all=False, name=None): #pylint: disable=redefined-builtin """Get job IDs - + Args: - all (bool): If True, use ``qstat`` to query all user jobs. Else, check + all (bool): If True, use ``qstat`` to query all user jobs. Else, check ``PBS_JOBID`` environment variable for ID of current job. - + name (str): If all==True, use name to filter results. - + Returns: One of str, List(str), or None: - Returns a str if all==False and ``PBS_JOBID`` exists, a List(str) + Returns a str if all==False and ``PBS_JOBID`` exists, a List(str) if all==True, else None. - + """ if all or name is not None: jobid = [] @@ -166,7 +167,7 @@ def job_rundir(jobid): Args: jobid (str or List(str)): IDs of jobs to get the run directory - + Returns: dict: A dict, with id:rundir pairs. @@ -192,11 +193,11 @@ def job_status(jobid=None): IDs of jobs to query for status. None for all user jobs. Returns: - + dict of dict: - + The outer dict uses jobid as key; the inner dict contains: - + ================ ====================================================== "name" Job name "nodes" Number of nodes @@ -208,7 +209,7 @@ def job_status(jobid=None): "starttime" None if not started, else seconds since epoch as int "completiontime" None if not completed, else seconds since epoch as int ================ ====================================================== - + """ status = dict() @@ -301,18 +302,18 @@ def job_status(jobid=None): return status -def submit(substr, write_submit_script=False): +def submit(substr, write_submit_script=None): """Submit a job using ``qsub``. Args: substr (str): The submit script string - write_submit_script (bool, optional): If true, submit via file skipping + write_submit_script (bool, optional): If true, submit via file skipping lines containing '#PBS -N'; otherwise, submit via commandline. If not specified, uses ``prisms_jobs.config['write_submit_script']``. - + Returns: str: ID of submitted job - + Raises: JobsError: If a submission error occurs """ @@ -326,8 +327,8 @@ def submit(substr, write_submit_script=False): r"""Error in pbs.misc.submit(). Jobname ("#PBS\s+-N\s+(.*)\s") not found in submit string.""") if write_submit_script is None: - write_submit_script = prisms_jobs.config['write_submit_script'] - + write_submit_script = prisms_jobs.config.write_submit_script() + if write_submit_script: if os.path.exists(jobname): index = 0 @@ -352,39 +353,39 @@ def submit(substr, write_submit_script=False): def delete(jobid): """``qdel`` a PBS job. - + Args: jobid (str): ID of job to delete - + Returns: int: ``qdel`` returncode - + """ stdout, stderr, returncode = run(["qdel", jobid]) #pylint: disable=unused-variable return returncode def hold(jobid): """``qhold`` a job. - + Args: jobid (str): ID of job to hold - + Returns: int: ``qhold`` returncode - + """ stdout, stderr, returncode = run(["qhold", jobid]) #pylint: disable=unused-variable return returncode def release(jobid): """``qrls`` a job. - + Args: jobid (str): ID of job to release - + Returns: int: ``qrls`` returncode - + """ stdout, stderr, returncode = run(["qrls", jobid]) #pylint: disable=unused-variable return returncode @@ -395,7 +396,7 @@ def alter(jobid, arg): Args: jobid (str): ID of job to alter arg (str): 'arg' is a scontrol command option string. For instance, "-a 201403152300.19" - + Returns: int: ``qalter`` returncode """ @@ -408,12 +409,12 @@ def read(job, qsubstr): #pylint: disable=too-many-branches, too-many-statemen * Will read many but not all valid PBS scripts. * Will ignore any arguments not included in prisms_jobs.Job()'s attributes. - * Will add default optional arguments (i.e. ``-A``, ``-a``, ``-l pmem=(.*)``, + * Will add default optional arguments (i.e. ``-A``, ``-a``, ``-l pmem=(.*)``, ``-l qos=(.*)``, ``-M``, ``-m``, ``-p``, ``"Auto:"``) if not found. - * Will ``exit()`` if required arguments (``-N``, ``-l walltime=(.*)``, + * Will ``exit()`` if required arguments (``-N``, ``-l walltime=(.*)``, ``-l nodes=(.*):ppn=(.*)``, ``-q``, ``cd $PBS_O_WORKDIR``) not found. * Will always include ``-V`` - + Args: qsubstr (str): A submit script as a string @@ -558,4 +559,3 @@ def read(job, qsubstr): #pylint: disable=too-many-branches, too-many-statemen sys.exit() # end if # end def - From a3a60eb175599496d9427525e7e28bb38b3b0653 Mon Sep 17 00:00:00 2001 From: bpuchala Date: Tue, 26 Feb 2019 09:10:10 -0500 Subject: [PATCH 4/5] add slurm constraint option --- prisms_jobs/job.py | 56 ++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/prisms_jobs/job.py b/prisms_jobs/job.py index e5575fb..238f048 100644 --- a/prisms_jobs/job.py +++ b/prisms_jobs/job.py @@ -21,11 +21,11 @@ class Job(object): #pylint: disable=too-many-instance-attributes Args: substr (str): A submit script as a string. name (str): Job name. Ex: ``"myjob-0"`` - - The name specified may be up to and including 15 characters + + The name specified may be up to and including 15 characters in length. It must consist of printable, non white space characters - with the first character alphabetic. - + with the first character alphabetic. + account (str): Account name. Ex: ``"prismsproject_fluxoe"`` nodes (int): Number of nodes. Ex: 2 ppn (int): Processors per node. Ex: 16 @@ -33,46 +33,46 @@ class Job(object): #pylint: disable=too-many-instance-attributes pmem (str): Memory requsted. Ex: ``"3800mb"`` qos (str): Ex: ``"flux"`` queue (str): Ex: ``"fluxoe"`` - + exetime (str): Time after which the job is eligible for execution. Ex: ``"1100"`` - + Has the form: ``[[[[CC]YY]MM]DD]hhmm[.SS]`` - Create using ``prisms_jobs.misc.exetime(deltatime)``, where deltatime + Create using ``prisms_jobs.misc.exetime(deltatime)``, where deltatime is a ``[[[DD:]MM:]HH:]SS`` string. - + message (str): When to send email about the job. Ex: ``"abe"`` - + The mail_options argument is a string which consists of either the single character ``"n"``, or one or more of the characters ``"a"``, ``"b"``, and ``"e"``. - + If the character ``"n"`` is specified, no normal mail is sent. Mail for job cancels and other events outside of normal job processing are still sent. - + For the letters ``"a"``, ``"b"``, and ``"e"``: - + === === a mail is sent when the job is aborted by the batch system. b mail is sent when the job begins execution. e mail is sent when the job terminates. === === - + email (str): Where to send notifications. Ex: ``"jdoe@umich.edu"`` - + The email string is of the form: ``user[@host][,user[@host],...]`` - + priority (str): Priority ranges from (low) -1024 to (high) 1023. Ex: ``"-200"`` - + command (str): String with command to run by script. Ex: ``"echo \"hello\" > test.txt"`` - + auto (bool, optional, Default=False): - + Indicates an automatically re-submitting job. Ex: ``True`` - - Only set to True if the command uses this prisms_jobs module to set - itself as completed when it is completed. Otherwise, you may submit + + Only set to True if the command uses this prisms_jobs module to set + itself as completed when it is completed. Otherwise, you may submit it extra times leading to wasted resources and overwritten data. - + Attributes: name (str): Job name. Ex: ``"myjob-0"`` account (str): Account name. Ex: ``"prismsproject_fluxoe"`` @@ -86,15 +86,16 @@ class Job(object): #pylint: disable=too-many-instance-attributes message (str): When to send email about the job. Ex: ``"abe"`` email (str): Where to send notifications. Ex: ``"jdoe@umich.edu"`` priority (str): Priority ranges from (low) -1024 to (high) 1023. Ex: ``"-200"`` + constraint (str): Constraint. Ex: ``"haswell"`` command (str): String with command to run by script. Ex: ``"echo \"hello\" > test.txt"`` auto (bool): Indicates an automatically re-submitting job. Ex: ``True`` - + """ def __init__(self, name="STDIN", account=None, nodes=None, ppn=None, walltime=None, #pylint: disable=too-many-arguments, too-many-locals pmem=None, qos=None, queue=None, exetime=None, message="a", email=None, - priority="0", command=None, auto=False, substr=None): + priority="0", constraint=None, command=None, auto=False, substr=None): if substr != None: self.read(substr) @@ -154,6 +155,9 @@ def __init__(self, name="STDIN", account=None, nodes=None, ppn=None, walltime=No # Priority ranges from (low) -1024 to (high) 1023 self.priority = priority + # Constraint (str): Ex: ``"haswell"`` + self.constraint = constraint + # text string with command to run self.command = command @@ -193,7 +197,7 @@ def submit(self, add=True, dbpath=None): Args: add (bool): Should this job be added to the JobDB database? dbpath (str): Specify a non-default JobDB database - + Raises: prisms_jobs.JobsError: If error submitting the job. @@ -222,5 +226,3 @@ def read(self, qsubstr): #pylint: disable=too-many-branches, too-many-stateme """ config.software().read(self, qsubstr) - - From 7d3f33c39eb12d5f6b572aa9ac5c263a8a012146 Mon Sep 17 00:00:00 2001 From: bpuchala Date: Wed, 17 Apr 2019 22:46:46 -0400 Subject: [PATCH 5/5] bump version to 4.0.2 --- README.md | 60 ++++++++++++++--------------- conda-recipes/prisms_jobs/meta.yaml | 4 +- doc/source/conf.py | 5 +-- doc/source/install.rst | 19 ++++----- prisms_jobs/__init__.py | 6 +-- 5 files changed, 43 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 6222828..6d15e37 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ prisms-jobs =========== -A Python package and set of scripts used for submitting and managing -cluster jobs via PBS (TORQUE) or Slurm. +A Python package and set of scripts used for submitting and managing +cluster jobs via PBS (TORQUE) or Slurm. -This code is developed by the PRedictive Integrated Structural Materials Science Center (PRISMS), at the University of Michigan, which is supported by the U.S. Department of Energy, Office of Basic Energy Sciences, Division of Materials Sciences and Engineering under Award #DE-SC0008637, and the Van der Ven group, originally at the University of Michigan and +This code is developed by the PRedictive Integrated Structural Materials Science Center (PRISMS), at the University of Michigan, which is supported by the U.S. Department of Energy, Office of Basic Energy Sciences, Division of Materials Sciences and Engineering under Award #DE-SC0008637, and the Van der Ven group, originally at the University of Michigan and currently at the University of California Santa Barbara. ## Overview -When submitted through the ``prisms_jobs`` Python package or the included scripts, -cluster jobs are stored in a SQLite jobs database. This allows for convenient -monitoring and searching of submitted jobs. +When submitted through the ``prisms_jobs`` Python package or the included scripts, +cluster jobs are stored in a SQLite jobs database. This allows for convenient +monitoring and searching of submitted jobs. It is often necessary to submit multiple jobs until a particular task is complete, -whether due to walltime or other limitations. ``prisms_jobs`` distinguishes and +whether due to walltime or other limitations. ``prisms_jobs`` distinguishes and tracks both individual "jobstatus" ('R', 'Q', 'C', 'E', etc.) and "taskstatus". -Jobs marked as 'auto' can be automatically or easily resubmitted until the +Jobs marked as 'auto' can be automatically or easily resubmitted until the "taskstatus" is "Complete". Possible values for "taskstatus" are: @@ -32,12 +32,12 @@ Possible values for "taskstatus" are: |"Aborted" |The job and task have been aborted. | -Jobs are marked 'auto' either by submitting through the python class ``prisms_jobs.Job`` -with the attribute ``auto=True``, or by submitting a script which contains +Jobs are marked 'auto' either by submitting through the python class ``prisms_jobs.Job`` +with the attribute ``auto=True``, or by submitting a script which contains the line ``#auto=True`` using the included ``psub`` command line program. -Jobs can be monitored using the command line program ``pstat``. All 'auto' jobs -which have stopped can be resubmitted using ``pstat --continue``. In this case, +Jobs can be monitored using the command line program ``pstat``. All 'auto' jobs +which have stopped can be resubmitted using ``pstat --continue``. In this case, 'continuation_jobid' is set with the jobid for the next job in the series of jobs comprising a task. @@ -58,15 +58,15 @@ Example screen shot: ------------ ------------------------ ----- ----- ------------ - ------------ ------------------------ - ------------ 11791026 taskmaster 1 1 0:01:00:00 W 0:01:00:00 Untracked 0 - -Additionally, when scheduling periodic jobs is not allowed other ways, the -``taskmaster`` script can fully automate this process. ``taskmaster`` executes +Additionally, when scheduling periodic jobs is not allowed other ways, the +``taskmaster`` script can fully automate this process. ``taskmaster`` executes ``pstat --continue`` and then resubmits itself to execute again periodically. -A script marked 'auto' should check itself for completion and when reached execute -``pstat --complete $JOBID --force`` in bash, or ``prisms_jobs.complete_job()`` in Python. If an 'auto' job script does not set its taskstatus to "Complete" it may continue +A script marked 'auto' should check itself for completion and when reached execute +``pstat --complete $JOBID --force`` in bash, or ``prisms_jobs.complete_job()`` in Python. If an 'auto' job script does not set its taskstatus to "Complete" it may continue to be resubmitted indefinitely. -Jobs not marked 'auto' are shown with the status "Check" in ``pstat`` until the user +Jobs not marked 'auto' are shown with the status "Check" in ``pstat`` until the user marks them as "Complete". @@ -77,11 +77,11 @@ Using ``pip``: pip install prisms-jobs or, to install in your user directory: - + pip install --user prisms-jobs - + If installing to a user directory, you may need to set your PATH to find the installed scripts. This can be done using: - + export PATH=$PATH:`python -m site --user-base`/bin @@ -99,20 +99,20 @@ If installing to a user directory, you may need to set your PATH to find the ins git clone https://github.com/prisms-center/prisms_jobs.git cd prisms_jobs -2. Checkout the branch/tag containing the version you wish to install. Latest is ``v4.0.1``: +2. Checkout the branch/tag containing the version you wish to install. Latest is ``v4.0.2``: - git checkout v4.0.1 + git checkout v4.0.2 2. From the root directory of the repository: pip install . - + or, to install in your user directory: - + pip install --user . - + If installing to a user directory, you may need to set your PATH to find the installed scripts. This can be done using: - + export PATH=$PATH:`python -m site --user-base`/bin @@ -123,15 +123,15 @@ See the [docs](https://prisms-center.github.io/prisms_jobs_docs/). ## License -This directory contains the prisms_jobs Python package and related scripts developed +This directory contains the prisms_jobs Python package and related scripts developed by the PRISMS Center at the University of Michigan, Ann Arbor, USA, and -the Van der Ven group, originally at the University of Michigan and +the Van der Ven group, originally at the University of Michigan and currently at the University of California Santa Barbara. (c) 2014 The Regents of the University of Michigan (c) 2015 The Regents of the University of California -PRISMS Center http://prisms.engin.umich.edu +PRISMS Center http://prisms.engin.umich.edu contact: casm-developers@lists.engr.ucsb.edu This code is a free software; you can use it, redistribute it, @@ -140,5 +140,3 @@ License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. Please see the file LICENSE for details. - - diff --git a/conda-recipes/prisms_jobs/meta.yaml b/conda-recipes/prisms_jobs/meta.yaml index fb70e4a..ed80272 100644 --- a/conda-recipes/prisms_jobs/meta.yaml +++ b/conda-recipes/prisms_jobs/meta.yaml @@ -1,9 +1,9 @@ package: name: prisms-jobs - version: "4.0.1" + version: "4.0.2" source: - git_rev: v4.0.1 + git_rev: v4.0.2 git_url: https://github.com/prisms-center/prisms_jobs.git requirements: diff --git a/doc/source/conf.py b/doc/source/conf.py index 3a57aa0..f1a301d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -86,7 +86,7 @@ # The short X.Y version. version = u'4.0' # The full version, including alpha/beta/rc tags. -release = u'4.0.1' +release = u'4.0.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -196,6 +196,3 @@ author, 'prisms_jobs', 'One line description of project.', 'Miscellaneous'), ] - - - diff --git a/doc/source/install.rst b/doc/source/install.rst index abc063f..480f22f 100644 --- a/doc/source/install.rst +++ b/doc/source/install.rst @@ -16,8 +16,8 @@ or, to install in your user directory: :: pip install --user prisms-jobs - -If installing to a user directory, you may need to set your PATH to find the + +If installing to a user directory, you may need to set your PATH to find the installed scripts. This can be done using: :: @@ -45,30 +45,27 @@ Install from source git clone https://github.com/prisms-center/prisms_jobs.git cd prisms_jobs -2. Checkout the branch/tag containing the version you wish to install. Latest is ``v4.0.1``: +2. Checkout the branch/tag containing the version you wish to install. Latest is ``v4.0.2``: :: - git checkout v4.0.1 + git checkout v4.0.2 3. From the root directory of the repository: :: pip install . - + or, to install in your user directory: :: pip install --user . - -If installing to a user directory, you may need to set your ``PATH`` to find the + +If installing to a user directory, you may need to set your ``PATH`` to find the installed scripts. This can be done using: -:: +:: export PATH=$PATH:`python -m site --user-base`/bin - - - diff --git a/prisms_jobs/__init__.py b/prisms_jobs/__init__.py index cce2d03..2973645 100644 --- a/prisms_jobs/__init__.py +++ b/prisms_jobs/__init__.py @@ -9,17 +9,17 @@ def __init__(self, jobid, msg): def __str__(self): return self.jobid + ": " + self.msg - + # import into 'prisms_jobs' from prisms_jobs.job import Job from prisms_jobs.jobdb import JobDB, JobDBError, EligibilityError, complete_job, error_job -__version__ = '4.0.1' +__version__ = '4.0.2' __all__ = [ 'Job', 'JobDB', 'JobsError', 'JobDBError', - 'EligibilityError', + 'EligibilityError', 'complete_job', 'error_job']