diff --git a/README.rst b/README.rst
index a3ef16f..edf2bfa 100644
--- a/README.rst
+++ b/README.rst
@@ -16,7 +16,7 @@
lmodule (Lmod Module)
---------------------
-lmodule is a Python API for `Lmod `_ module system. The API can be used
+lmodule is a Python 3 API for `Lmod `_ module system. The API can be used
end-users and system-administrators. End-users can utilize the API for interacting with module system to run
their application. System Administrators can utilize the API for testing their Software Stack and retrieve
output from the Lmod spider command.
diff --git a/docs/index.rst b/docs/index.rst
index a01f75e..6e874af 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -7,10 +7,28 @@ Welcome to lmodule documentation!
===================================
`lmodule `_ is a Python API for `Lmod `_
-module system. lmodule was part of `buildtest `_ and we found that
-a Python API for the Lmod system would benefit other's who may not need to use buildtest.
+module system. lmodule was originally part of `buildtest `_ and
+we decided that it could benefit the entire community for folks interested in using the API but not relying on buildtest.
-The name **lmodule** was decided by taking **Lmod** + **module** = **lmodule**.
+What is lmodule?
+-----------------
+
+Lmodule is a Python 3 API that allows you to interact with the **module** command provided by Lmod in a programmatic way.
+The API comes with three classes:
+
+- Module: This class emulates the ``module`` command
+
+- Spider: This class runs the Lmod ``spider`` command to retrieve all module records in json
+
+- ModuleLoadTest: This class automates ``module load`` of one or more module trees
+
+Why use lmodule?
+----------------
+
+Here are few reasons why you would want to use lmodule
+
+1. Currently, there is no Python API for Lmod, however there is a python interface ``LMOD_CMD python`` by Lmod that requires parsing and output is cryptic.
+2. Automates ``module load`` test for each module in one or more module trees (Software Stack). This can be used to spot faulty modules in a large software stack. This type of test is meant to run using CI tool for continuous testing of module stack.
.. toctree::
diff --git a/lmod/module.py b/lmod/module.py
index 98b6ec2..88e7719 100755
--- a/lmod/module.py
+++ b/lmod/module.py
@@ -2,10 +2,11 @@
import subprocess
def get_user_collections():
- """Get all Lmod user collections that is retrieved by running ``module -t savelist``.
+ """Get all user collections that is retrieved by running ``module -t savelist``. The output
+ is of type ``list`` and each entry in list is the name of the user collection.
- :return: Return all module collections
- :rtype: list
+ :return: Return all user collections
+ :rtype: list
"""
collections = "module -t savelist"
@@ -22,21 +23,41 @@ def get_user_collections():
class Module:
- """Class declaration for Module class"""
+ """This is the class declaration of Module class which emulates the ``module`` command
+ provided by Lmod
+
+ Public Methods:
+ ---------------
+ avail: implements ``module avail`` option
+ version: gets Lmod version by reading LMOD_VERSION
+ is_avail: implements ``module is-avail`` option
+ get_command: gets the ``module load`` command based on modules passed to the class
+ test_modules: test the ``module load`` command based on modules passed to the class
+ save: implements ``module save`` option
+ describe: implements ``module describe`` option
+ get_collection: gets the ``module restore`` command with the specified collection name
+ test_collection: test the ``module restore`` command with the specified collection name
+ """
def __init__(self, modules=None, purge=True, force=False, debug=False):
- """Initialize method for Module class.
-
- :param modules: list of modules
- :param purge: boolean to control whether to purge modules before loading
- :param force: boolean to control whether to force purge modules before loading
- :param debug: debug mode for troubleshooting
-
- :type modules: list
- :type purge: bool
- :type force: bool
- :type debug: bool
+ """Initialize method for Module class. This class can accept module names that
+ can be used to load or test modules. You can tweak the behavior of how module
+ command is generated such as purging of force purge modules. The debug option
+ can be useful for troubleshooting.
+
+ Parameters:
+ -----------
+ :param modules: list of modules
+ :param purge: boolean to control whether to purge modules before loading
+ :param force: boolean to control whether to force purge modules before loading
+ :param debug: debug mode for troubleshooting
+
+ :type modules: list, optional
+ :type purge: bool, optional
+ :type force: bool, optional
+ :type debug: bool, optional
"""
+
self.debug = debug
self.modules = modules
@@ -71,13 +92,17 @@ def __init__(self, modules=None, purge=True, force=False, debug=False):
self.module_load_cmd = ["module purge &&"] + self.module_load_cmd
def avail(self,name=None):
- """This method implements the ``module avail`` command.
+ """This method implements the ``module avail`` command. The output of module avail will return available
+ modules in the system. The output is returned as a list using the ``module -t avail`` which presents the
+ output in a single line per module.
- :param name: argument passed to ``module av``. This is used for showing what modules are available
- :type name: str
+ Parameters:
+ -----------
+ :param name: argument passed to ``module avail``. This is used for showing what modules are available
+ :type name: str, optional
- :return: Return output of ``module avail`` as a list
- :rtype: list
+ :return: Return output of ``module avail`` as a list
+ :rtype: list
"""
cmd = "module -t avail"
@@ -96,22 +121,27 @@ def avail(self,name=None):
return ret.stdout.split()
def version(self):
- """Get Lmod version by getting value from environment ``LMOD_VERSION``
+ """Get Lmod version by reading environment variable ``LMOD_VERSION`` and return as a string
- :return: Return the Lmod version
- :rtype: str
+ :return: Return the Lmod version as a string
+ :rtype: str
"""
+
return os.getenv("LMOD_VERSION") or None
def is_avail(self, name):
- """This method implements the ``module is-avail`` command.
+ """This method implements the ``module is-avail`` command which is used for checking if a module is available
+ before loading it. The return value is a 0 or 1.
- :param name: argument passed to ``module is-avail``. This is used for checking if module is available
- :type name: str
+ Parameters:
+ ------------
+ :param name: argument passed to ``module is-avail``. This is used for checking if module is available
+ :type name: str, required
- :return: Return output of ``module is-avail``. This checks if module is available and return code is a 0 or 1
- :rtype: int
+ :return: Return output of ``module is-avail``. This checks if module is available and return code is a 0 or 1
+ :rtype: int
"""
+
cmd = f"module is-avail {name}"
ret = subprocess.run(
@@ -125,20 +155,28 @@ def is_avail(self, name):
return ret.returncode
def get_command(self):
- """ Get the actual module load command that can be used to load the given modules.
+ """Get the actual module load command that can be used to load the given modules.
- :return: return the actual module load command
- :rtype: str
+ :return: return the actual module load command
+ :rtype: str
"""
return " ".join(self.module_load_cmd)
def test_modules(self, login=False):
- """ Test all specified modules by loading them using ``module load``.
+ """Test all modules passed to Module class by loading them using ``module load``. The default behavior
+ is to run the command in a sub-shell but this can be changed to run in a new login shell if ``login=True`` is
+ specified. The return value is a return code (type ``int``) of the ``module load`` command.
+
+ Parameters:
+ -----------
+ :param login: When ``login=True`` is set, it will run the test in a login shell, the default is to run in a sub-shell
+ :type login: bool, optional
- :return: return code of ``module load`` command
- :rtype: int
+ :return: return code of ``module load`` command
+ :rtype: int
"""
+
cmd_executed = self.get_command()
# run test in login shell
@@ -161,10 +199,16 @@ def test_modules(self, login=False):
return ret.returncode
def save(self, collection="default"):
- """Save active modules into a module collection.
-
- :param collection: collection name to save modules. If none specified, ``default`` is the collection.
- :type collection: str
+ """Save modules specified in Module class into a user collection. This implements the ``module save`` command
+ for active modules. In this case, we are saving modules that were passed to the Module class. If no argument
+ is specified, we will save to ``default`` collection, but user can specify a collection name, in that case
+ we are running ``module save ``. The collection name must be of type ``str`` in order for
+ this to work, otherwise an exception of ``TypeError`` will be raised.
+
+ Paramters:
+ -----------
+ :param collection: collection name to save modules. If none specified, ``default`` is the collection.
+ :type collection: str, optional
"""
# raise TypeError exception if collection is not a string type since that is required
@@ -191,10 +235,15 @@ def save(self, collection="default"):
print(ret.stdout)
def describe(self, collection="default"):
- """Show content of a module collection.
-
- :param collection: name of module collection
- :type collection: str
+ """Show content of a user collection and implements the command ``module describe``. By default, if no argument
+ is specified it will resort to showing contents of ``default`` collection. One can pass a collection name
+ which must be of type ``str`` that is the user collection name. Internally it will run ``module describe ``.
+ If collection name is not of type ``str``, then an exception of ``TypeError`` will be raised.
+
+ Parameters:
+ -----------
+ :param collection: name of user collection to show.
+ :type collection: str, optional
"""
# raise TypeError exception if collection is not a string type since that is required
@@ -219,14 +268,20 @@ def describe(self, collection="default"):
print(ret.stdout)
def get_collection(self, collection="default"):
- """Return the command to restore a collection.
-
- :param collection: collection name to restore
- :type collection: str
-
- :return: return the ``module restore`` command with the collection name
- :rtype: str
+ """Return the module command to restore a collection. If no argument is specified, it will resort to the ``default``
+ collection, otherwise one can specify a collection name of type ``str``. The output will be of type ``str``
+ such as ``module restore default``. If argument to class is not of type ``str`` then an exception of type
+ ``TypeError`` will be raised.
+
+ Parameters:
+ -----------
+ :param collection: collection name to restore
+ :type collection: str, optional
+
+ :return: return the ``module restore`` command with the collection name
+ :rtype: str
"""
+
# raise error if collection is not a string
if not isinstance(collection, str):
raise TypeError(f"Type Error: {collection} is not of type string")
@@ -234,13 +289,21 @@ def get_collection(self, collection="default"):
return f"module restore {collection}"
def test_collection(self, collection="default"):
- """Test the module collection by running ``module restore ``.
- :param collection: collection name to test
- :type collection: str
-
- :return: return code of ``module restore`` against the collection name
- :rtype: int
+ """Test the user collection by running ``module restore`` against a collection name. This is useful, to test a
+ user collection is working before using it in your scripts. If no argument is specified, it will test the
+ ``default`` collection. One can specify a user collection name which must of be of type ``str`` and it must
+ exist. The output will be a return code of the ``module restore`` command which would be of type ``int``.
+ If argument to method is not of type ``str`` an exception of ``TypeError`` will be raised.
+
+ Parameters:
+ -----------
+ :param collection: collection name to test
+ :type collection: str, optional
+
+ :return: return code of ``module restore`` against the collection name
+ :rtype: int
"""
+
# raise error if collection is not a string
if not isinstance(collection, str):
raise TypeError(f"Type Error: {collection} is not of type string")
diff --git a/lmod/moduleloadtest.py b/lmod/moduleloadtest.py
index d5aa15d..42edabc 100755
--- a/lmod/moduleloadtest.py
+++ b/lmod/moduleloadtest.py
@@ -2,10 +2,12 @@
from lmod.module import Module
from lmod.spider import Spider
-
class ModuleLoadTest:
"""This is the class declaration of ModuleLoadTest. This class will automate module load
- test for all modules in one or more module tree (Software Stack).
+ test for all modules in one or more module tree (Software Stack) retrieved by Spider class. The output
+ of ``module load`` is a 0 or 1 that can be used to determine PASS/FAIL on a module. Each module test will attempt
+ to test the ``module load`` command using the ``Module().test_modules()``. In order to run tests properly,
+ MODULEPATH must be set in your environment.
"""
def __init__(
@@ -20,35 +22,37 @@ def __init__(
exclude=[],
login=False,
):
- """This is the initializer method that automates testing of modules
+ """This is the initializer method for ModuleLoadTest class.
+
+ Parameters:
+ -----------
+ :param tree: specify one or more module trees to test. The module tree must be root directory where modulefiles
+ are found. Use a colon ``:`` to define more than one module tree.
+ :type tree: str
+
+ :param purge: control whether to run ``module purge`` before loading each module
+ :type purge: bool
- :param tree: specify one or more module trees to test. The module tree must be root directory where modulefiles
- are found. Use a colon ``:`` to define more than one module tree.
- :type tree: str
+ :param force: control whether to run ``module --force purge`` before loading each module
+ :type purge: bool
- :param purge: control whether to run ``module purge`` before loading each module
- :type purge: bool
+ :param login: controls whether to run test in login shell when ``login=True``. By default tests are run in sub-shell.
+ :type purge: bool
- :param force: control whether to run ``module --force purge`` before loading each module
- :type purge: bool
-
- :param login: control whether to run test in login shell. Defaults to sub-shell when ``login=False``
- :type purge: bool
-
- :param count: control how many tests to run before exiting
- :type purge: int
+ :param count: control how many tests to run before exiting
+ :type purge: int
- :param name: filter modules by software name to test
- :type name: list
+ :param name: filter modules by software name to test
+ :type name: list
- :param include: specify a list of modules to **include** by full canonical name for testing
- :type purge: list
+ :param include: specify a list of modules to **include** by full canonical name for testing
+ :type purge: list
- :param exclude: specify a list of modules to **exclude** by full canonical name for testing
- :type purge: list
+ :param exclude: specify a list of modules to **exclude** by full canonical name for testing
+ :type purge: list
- :return: Result of module load test
- :rtype: None
+ :return: Result of module load test
+ :rtype: None
"""
# setting module tree to argument passed in or default to MODULEPATH
@@ -76,8 +80,10 @@ def __init__(
modules = filter_modules or modules
modulecount = 0
+
print(f"Testing the Following Module Trees: {self.tree}")
print("{:_<80}".format(""))
+
for module_name in modules:
module_cmd = Module(
module_name, purge=self.purge, force=self.force, debug=self.debug,
diff --git a/lmod/spider.py b/lmod/spider.py
index 8e2fac8..84a092a 100755
--- a/lmod/spider.py
+++ b/lmod/spider.py
@@ -2,15 +2,30 @@
import os
import subprocess
-
class Spider:
- """Class declaration of Spider class"""
+ """This is the class declaration of Spider class which emulates the spider tool provided by Lmod. The spider command
+ is typically used to build the spider cache in your site. We use the spider tool to fetch all module files.
+
+ Public Methods:
+ ---------------
+ get_trees: returns all module trees used with spider command
+ get_names: returns all top-level keys from spider which is the software name
+ get_modules: returns all full canonical module name.
+ get_parents: return all parent modules, parent modules are modules that set MODULEPATH to another tree.
+ get_all_versions: returns all versions of a specified module.
+ """
def __init__(self, tree=None):
- """Initialize method for Spider class.
+ """Initialize method for Spider class. The spider tool is provided by Lmod typically found in ``LMOD_DIR/spider``.
+ We are running ``spider -o spider-json ``. If no argument is specified, then we default to
+ MODULEPATH as the tree. One can pass one or more module tree separated by colon (``:``). The output will
+ be a json structure (``dict``) that is stored in class variable ``self.spider_content``.
+
- :param tree: User can specify a module tree to query from spider.
- :type tree: str
+ Parameters:
+ ------------
+ :param tree: User can specify one or more module trees to query from spider. Trees must be separated by colon (``:``)
+ :type tree: str, optional
"""
# set spider tree to value passed in to class or value of MODULEPATH
self.tree = tree or os.getenv("MODULEPATH")
@@ -23,19 +38,26 @@ def __init__(self, tree=None):
self.spider_content = json.loads(out)
def get_trees(self):
- """" Return module trees used in spider command
+ """"Return module trees used in spider command.
- :return: return module trees used for querying from spider
- :rtype: str
+ :return: return module trees used for querying from spider
+ :rtype: str
"""
return self.tree
def get_names(self, name=[]):
- """Return all keys from spider. This is the unique software names.
+ """Returns a list of software names which are found by returning the top-level key from json structure.
+ One can specify a list of module names to filter output.
- :return: return sorted list of all spider keys.
- :rtype: list
+ Parameters
+ -----------
+ :param name: a list of software name to filter output
+ :type name: list, optional
+
+ :return: return sorted list of all spider keys.
+ :rtype: list
"""
+
if name:
name_list = set(name).intersection(self.spider_content.keys())
return sorted(name_list)
@@ -43,10 +65,20 @@ def get_names(self, name=[]):
return sorted(list(self.spider_content.keys()))
def get_modules(self, name=[]):
- """Retrieve all module names from all module tree.
+ """Retrieve all full-canonical module names. This can be retrieved by fetching ``fullName`` key
+ in the json output. The full-canonical module name represents the actual module name. One can
+ filter output by passing a list of software names, if no argument is specified we will return all
+ modules. We ignore spider records that contain ``.version`` or ``.modulerc`` whih are not actual
+ modules.
- :return: returns a sorted list of all full canonical module name from all spider records.
- :rtype: dict
+ Parameters:
+ ------------
+
+ :param name: a list of software name to filter output
+ :type name: type, required
+
+ :return: returns a sorted list of all full canonical module name from all spider records.
+ :rtype: dict
"""
module_names = {}
@@ -64,13 +96,15 @@ def get_modules(self, name=[]):
def get_parents(self):
"""Return all parent modules from all spider trees. This will search all ``parentAA`` keys in spider
- content. The parent modules are used for setting MODULEPATH to other trees.
+ content. The parent modules are used for setting MODULEPATH to other trees. The parentAA is a nested list
+ containing one or more parent module combination required to load a particular module. The spider output
+ can contain several occurrences of parent modules in parentAA key so use a ``set`` to add unique parent modules.
- :return: sorted list of all parent modules.
- :rtype: list
+ :return: sorted list of all parent modules.
+ :rtype: list
"""
- # we only care about unique modules. parentAA is bound to have duplicate modules.
+ # we only care about unique modules. parentAA is bound to have many modules.
parent_set = set()
for module in self.get_names():
@@ -82,11 +116,16 @@ def get_parents(self):
return sorted(list(parent_set))
def get_all_versions(self, key):
- """Get all versions of a particular software name.
- :param key: name of software
- :type key: str
+ """Get all versions of a particular software name. This is can be retrieved by reading ``Version`` key
+ in spider output.
+
+ Parameters:
+ -----------
+ :param key: name of software
+ :type key: str, required
- :return: list of module name as versions
+ :return: list of module name as versions
+ :rtype: list
"""
# return empty list of key is not found