From eacbb9f22be8a6ddf8b4576eef122d541abb5a72 Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Tue, 19 Oct 2021 23:24:09 -0700 Subject: [PATCH 1/5] extended FSWTabDict class --- ait/core/cfg.py | 4 ++++ ait/core/cmd.py | 4 ++-- ait/core/dtype.py | 2 +- ait/core/newCmd.py | 10 ++++++++++ ait/core/newTable.py | 14 ++++++++++++++ ait/core/table.py | 4 ++++ ait/core/test/test_table.py | 1 + ait/core/util.py | 6 ++++++ ait/extCmdTest.py | 7 +++++++ ait/extTableTest.py | 8 ++++++++ 10 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 ait/core/newCmd.py create mode 100644 ait/core/newTable.py create mode 100644 ait/extCmdTest.py create mode 100644 ait/extTableTest.py diff --git a/ait/core/cfg.py b/ait/core/cfg.py index 43ab15eb..9d9481b3 100644 --- a/ait/core/cfg.py +++ b/ait/core/cfg.py @@ -168,8 +168,12 @@ def loadYAML (filename=None, data=None): if filename: data = open(filename, 'rt') + print(f"DATA: {data}") + config = yaml.load(data, Loader=yaml.Loader) + print(f"CONFIG: {config}") + if isinstance(data, IOBase): data.close() except IOError as e: diff --git a/ait/core/cmd.py b/ait/core/cmd.py index f2cfe84b..68f3d3e3 100644 --- a/ait/core/cmd.py +++ b/ait/core/cmd.py @@ -474,7 +474,7 @@ def load(self, content): else: stream = content - cmds = yaml.load(stream, Loader=yaml.Loader) + cmds = yaml.load(stream, Loader=yaml.FullLoader) cmds = handle_includes(cmds) for cmd in cmds: self.add(cmd) @@ -544,7 +544,7 @@ def YAMLCtor_include(loader, node): name = os.path.join(os.path.dirname(loader.name), node.value) data = None with open(name,'r') as f: - data = yaml.load(f) + data = yaml.load(f, Loader=yaml.FullLoader) return data yaml.add_constructor('!include' , YAMLCtor_include) diff --git a/ait/core/dtype.py b/ait/core/dtype.py index 3feb7f2b..6388d8dc 100644 --- a/ait/core/dtype.py +++ b/ait/core/dtype.py @@ -71,7 +71,7 @@ import sys import re -from ait.core import cmd, dmc, log, util +from ait.core import cmd, dmc, log, util, table # PrimitiveTypes diff --git a/ait/core/newCmd.py b/ait/core/newCmd.py new file mode 100644 index 00000000..ae2bfadc --- /dev/null +++ b/ait/core/newCmd.py @@ -0,0 +1,10 @@ +from ait.core import cmd, log + + +class NewCmd(cmd.Cmd): + def encode(self): + log.info('We are in an extension of cmd.encode() that will then call the regular cmd.encode().') + return super(NewCmd, self).encode() + + def custom(self): + log.info("In a custom() method defined in the NewCmd extended class.") diff --git a/ait/core/newTable.py b/ait/core/newTable.py new file mode 100644 index 00000000..a39c9b60 --- /dev/null +++ b/ait/core/newTable.py @@ -0,0 +1,14 @@ +from ait.core import table, log + + +class NewFSWTabDict(table.FSWTabDict): + def load(self): + log.info('Starting the table load() from the extended FSWTabDict class, using file: {}'.format(self.filename)) + return super(NewFSWTabDict, self).load() + + def create(self): + log.info('Starting the table create() from custom extension class') + return super(NewFSWTabDict, self).create() + + def custom(): + log.info("Test of a unique method defined in an extension.") diff --git a/ait/core/table.py b/ait/core/table.py index 13c9f74d..7ab0ce74 100644 --- a/ait/core/table.py +++ b/ait/core/table.py @@ -542,3 +542,7 @@ def decode_to_file(tbl_type, in_path, out_path): yaml.add_constructor("!FSWTable", YAMLCtor_FSWTabDefn) yaml.add_constructor("!FSWColumn", YAMLCtor_FSWColDefn) + +print(f'__name__: {__name__}') + +util.__init_extensions__(__name__, globals()) diff --git a/ait/core/test/test_table.py b/ait/core/test/test_table.py index df6bd5bb..1e36db0a 100644 --- a/ait/core/test/test_table.py +++ b/ait/core/test/test_table.py @@ -373,6 +373,7 @@ def test_table_duplicate_enum_load(self): tabdict = table.FSWTabDict(table_defn_path) assert len(cm.output) == 1 + class TestTableTimeHandling(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/ait/core/util.py b/ait/core/util.py index 1d677209..38601ef5 100755 --- a/ait/core/util.py +++ b/ait/core/util.py @@ -141,6 +141,8 @@ class name (*extname*). subsequent calls. """ def create(*args, **kwargs): + print("*****************") + print(create.cls) if create.cls is None: parts = extname.rsplit('.', 1) if len(parts) > 1: @@ -166,11 +168,15 @@ def create(*args, **kwargs): if extensions: extname = extensions.get(modname + '.' + clsname, None) + print(f"extname: {extname}") + if extname: cls = None values = modname, clsname, extname log.info('Replacing %s.%s with custom extension: %s' % values) + print('create' + clsname) + modsyms['create' + clsname] = createFunc(cls, extname) diff --git a/ait/extCmdTest.py b/ait/extCmdTest.py new file mode 100644 index 00000000..8b9d43f7 --- /dev/null +++ b/ait/extCmdTest.py @@ -0,0 +1,7 @@ +import ait.core.cmd as cmd +# import ait.core.log as log + +cmdDict = cmd.getDefaultDict() +no_op = cmdDict.create('NO_OP') +no_op.encode() +no_op.custom() diff --git a/ait/extTableTest.py b/ait/extTableTest.py new file mode 100644 index 00000000..61114685 --- /dev/null +++ b/ait/extTableTest.py @@ -0,0 +1,8 @@ +import ait.core.table as table + +# tab_dict = table.FSWTabDict("/Users/jehofman/AIT-Core/config/table.yaml") +# tab_dict.load(filename=tab_dict.filename) + +table_dict = table.getDefaultFSWTabDict() +table_dict.create('MyTable') +table_dict.load(filename=table_dict.filename) From a6dd09c66fcbcb0aa91e3b737f7bdacb761fed7c Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Wed, 20 Oct 2021 12:25:59 -0700 Subject: [PATCH 2/5] issue-376: update new branch --- ait/core/cmd.py | 5 +++-- ait/extTableTest.py | 1 + config/config.yaml | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ait/core/cmd.py b/ait/core/cmd.py index 68f3d3e3..04aa484d 100644 --- a/ait/core/cmd.py +++ b/ait/core/cmd.py @@ -474,7 +474,7 @@ def load(self, content): else: stream = content - cmds = yaml.load(stream, Loader=yaml.FullLoader) + cmds = yaml.load(stream, Loader=yaml.Loader) cmds = handle_includes(cmds) for cmd in cmds: self.add(cmd) @@ -544,9 +544,10 @@ def YAMLCtor_include(loader, node): name = os.path.join(os.path.dirname(loader.name), node.value) data = None with open(name,'r') as f: - data = yaml.load(f, Loader=yaml.FullLoader) + data = yaml.load(f, Loader=yaml.Loader) return data + yaml.add_constructor('!include' , YAMLCtor_include) yaml.add_constructor('!Command' , YAMLCtor_CmdDefn) yaml.add_constructor('!Argument', YAMLCtor_ArgDefn) diff --git a/ait/extTableTest.py b/ait/extTableTest.py index 61114685..18da3960 100644 --- a/ait/extTableTest.py +++ b/ait/extTableTest.py @@ -6,3 +6,4 @@ table_dict = table.getDefaultFSWTabDict() table_dict.create('MyTable') table_dict.load(filename=table_dict.filename) + diff --git a/config/config.yaml b/config/config.yaml index 3255260a..281e7c13 100755 --- a/config/config.yaml +++ b/config/config.yaml @@ -84,6 +84,11 @@ default: max_entity_id_length: 8 max_transaction_id_length: 8 + extensions: + ait.core.cmd.Cmd: ait.core.newCmd.NewCmd + ait.core.table.FSWTabDict: ait.core.newTable.NewFSWTabDict + + logging: name: ait hostname: yourCustomHostForLogging @@ -165,4 +170,5 @@ default: - stream: name: command_stream output: 3075 - command_subscriber: True \ No newline at end of file + command_subscriber: True + From 9fbae8b3689f0e374fb1dbbacad65f0cb646a081 Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Thu, 28 Oct 2021 13:56:19 -0700 Subject: [PATCH 3/5] item-376 extensions for table.py --- ait/core/cfg.py | 23 +++---- ait/core/cmd.py | 59 +++++++++++------ ait/core/newCmd.py | 10 --- ait/core/newTable.py | 14 ---- ait/core/table.py | 17 +++-- ait/core/util.py | 151 ++++++++++++++++++++++++++----------------- ait/extCmdTest.py | 7 -- ait/extTableTest.py | 9 --- config/config.yaml | 9 ++- 9 files changed, 153 insertions(+), 146 deletions(-) delete mode 100644 ait/core/newCmd.py delete mode 100644 ait/core/newTable.py delete mode 100644 ait/extCmdTest.py delete mode 100644 ait/extTableTest.py diff --git a/ait/core/cfg.py b/ait/core/cfg.py index 9d9481b3..43b2e7a3 100644 --- a/ait/core/cfg.py +++ b/ait/core/cfg.py @@ -40,13 +40,15 @@ PATH_KEYS = 'directory', 'file', 'filename', 'path', 'pathname' def expandConfigPaths (config, prefix=None, datetime=None, pathvars=None, parameter_key='', *keys): - """Updates all relative configuration paths in dictionary config, + """ + Updates all relative configuration paths in dictionary config, which contain a key in keys, by prepending prefix. If keys is omitted, it defaults to 'directory', 'file', 'filename', 'path', 'pathname'. See util.expandPath(). + """ if len(keys) == 0: keys = PATH_KEYS @@ -117,7 +119,6 @@ def replaceVariables(path, datetime=None, pathvars=None): # get the list of possible variable values value_list = v if type(v) is list else [ v ] - # create temp_list for now temp_list = [] @@ -168,12 +169,8 @@ def loadYAML (filename=None, data=None): if filename: data = open(filename, 'rt') - print(f"DATA: {data}") - config = yaml.load(data, Loader=yaml.Loader) - print(f"CONFIG: {config}") - if isinstance(data, IOBase): data.close() except IOError as e: @@ -193,14 +190,12 @@ def merge (d, o): return d - class AitConfigError(Exception): """Raised when a AIT configuration parameter is present, but is in some way incorrect.""" pass - class AitConfigMissing(Exception): """Raised when a AIT configuration parameter is missing.""" @@ -211,9 +206,9 @@ def __init__(self, param): self.param = param - class AitConfig (object): - """AitConfig + """ + AitConfig A AitConfig object holds configuration parameters read from a YAML configuration file. The YAML data structure has three levels @@ -222,6 +217,7 @@ class AitConfig (object): NOTE: The platform string is Python's sys.platform, i.e. 'linux2', 'darwin', 'win32'. + """ _ROOT_DIR = os.path.abspath(os.environ.get('AIT_ROOT', os.getcwd())) @@ -337,12 +333,13 @@ def _datapaths(self): return paths def reload (self, filename=None, data=None): - """Reloads the a AIT configuration. - + """ + Reloads the a AIT configuration. The AIT configuration is automatically loaded when the AIT package is first imported. To replace the configuration, call reload() (defaults to the current config.filename) or reload(new_filename). + """ if data is None and filename is None: filename = self._filename @@ -368,7 +365,6 @@ def reload (self, filename=None, data=None): else: self._config = { } - def get (self, name, default=None): """Returns the attribute value *AitConfig.name* or *default* if name does not exist. @@ -397,7 +393,6 @@ def get (self, name, default=None): return config[tail] if tail in config else default - def getDefaultFilename(self): if 'AIT_CONFIG' in os.environ: filename = os.path.abspath(os.environ.get('AIT_CONFIG')) diff --git a/ait/core/cmd.py b/ait/core/cmd.py index 04aa484d..b9ac7953 100644 --- a/ait/core/cmd.py +++ b/ait/core/cmd.py @@ -17,6 +17,7 @@ The ait.core.cmd module provides commands and command dictionaries. Dictionaries contain command and argument definitions. + """ import os @@ -43,6 +44,7 @@ class ArgDefn(json.SlotSerializer, object): A fixed argument (fixed=True) defines a fixed bit pattern in that argument's byte position(s). + """ __slots__ = [ "name", "desc", "units", "_type", "bytes", "_enum", "range", @@ -96,8 +98,10 @@ def startbit(self): return self.slice().start % 2 * 8 def decode(self, bytes): - """Decodes the given bytes according to this AIT Argument + """ + Decodes the given bytes according to this AIT Argument Definition. + """ value = self.type.decode(bytes) if self._enum is not None: @@ -108,18 +112,22 @@ def decode(self, bytes): return value def encode(self, value): - """Encodes the given value according to this AIT Argument + """ + Encodes the given value according to this AIT Argument Definition. + """ if type(value) == str and self.enum and value in self.enum: value = self.enum[value] return self.type.encode(value) if self.type else bytearray() def slice(self, offset=0): - """Returns a Python slice object (e.g. for array indexing) indicating + """ + Returns a Python slice object (e.g. for array indexing) indicating the start and stop byte position of this Command argument. The start and stop positions may be translated by the optional byte offset. + """ if type(self.bytes) is int: start = self.bytes @@ -131,9 +139,11 @@ def slice(self, offset=0): return slice(start + offset, stop + offset) def validate(self, value, messages=None): - """Returns True if the given Argument value is valid, False otherwise. + """ + Returns True if the given Argument value is valid, False otherwise. Validation error messages are appended to an optional messages array. + """ valid = True primitive = value @@ -163,16 +173,19 @@ def log(msg): return valid - class Cmd(object): - """Cmd - Command + """ + Cmd - Command Commands reference their Command Definition and may contain arguments. + """ def __init__(self, defn, *args, **kwargs): - """Creates a new AIT Command based on the given command + """ + Creates a new AIT Command based on the given command definition and command arguments. A Command may be created with either positional or keyword arguments, but not both. + """ self.defn = defn @@ -193,7 +206,6 @@ def __init__(self, defn, *args, **kwargs): self.args = args self._unrecognized = kwargs - def __repr__(self): return self.defn.name + " " + " ".join([str(a) for a in self.args]) @@ -223,7 +235,8 @@ def argdefns(self): return self.defn.argdefns def encode(self, pad=106): - """Encodes this AIT command to binary. + """ + Encodes this AIT command to binary. If pad is specified, it indicates the maximum size of the encoded command in bytes. If the encoded command is less than pad, the @@ -233,6 +246,7 @@ def encode(self, pad=106): (128 bytes) with 11 words (22 bytes) of CCSDS overhead (SSP 52050J, Section 3.2.3.4). This leaves 53 words (106 bytes) for the command itself. + """ opcode = struct.pack('>H', self.defn.opcode) offset = len(opcode) @@ -255,21 +269,24 @@ def encode(self, pad=106): return encoded def validate(self, messages=None): - """Returns True if the given Command is valid, False otherwise. + """ + Returns True if the given Command is valid, False otherwise. Validation error messages are appended to an optional messages array. + """ return self.defn.validate(self, messages) - class CmdDefn(json.SlotSerializer, object): - """CmdDefn - Command Definition + """ + mdDefn - Command Definition Command Definitions encapsulate all information required to define a single command. This includes the command name, its opcode, subsystem, description and a list of argument definitions. Name and opcode are required. All others are optional. + """ __slots__ = ( 'name', '_opcode', 'subsystem', 'ccsds', 'title', 'desc', 'argdefns' ) @@ -287,32 +304,37 @@ def __init__(self, *args, **kwargs): if self.argdefns is None: self.argdefns = [] - def __repr__(self): return util.toRepr(self) @property def args (self): - """The argument definitions to this command (excludes fixed + """ + The argument definitions to this command (excludes fixed arguments). + """ return filter(lambda a: not a.fixed, self.argdefns) @property def nargs(self): - """The number of arguments to this command (excludes fixed + """ + The number of arguments to this command (excludes fixed arguments). + """ return len(list(self.args)) @property def nbytes(self): - """The number of bytes required to encode this command. + """ + The number of bytes required to encode this command. Encoded commands are comprised of a two byte opcode, followed by a one byte size, and then the command argument bytes. The size indicates the number of bytes required to represent command arguments. + """ return len(self.opcode) + 1 + sum(arg.nbytes for arg in self.argdefns) @@ -376,7 +398,6 @@ def validate(self, cmd, messages=None): return valid - class CmdDict(dict): """CmdDict @@ -412,7 +433,6 @@ def add(self, defn): log.error(msg) raise util.YAMLError(msg) - def create(self, name, *args, **kwargs): """Creates a new AIT command with the given arguments.""" tokens = name.split() @@ -434,7 +454,6 @@ def create(self, name, *args, **kwargs): return createCmd(defn, *args, **kwargs) - def decode(self, bytes): """Decodes the given bytes according to this AIT Command Definition. @@ -486,7 +505,6 @@ def toJSON(self): return { name: defn.toJSON() for name, defn in self.items() } - def getDefaultCmdDict(reload=False): return getDefaultDict(reload=reload) @@ -539,6 +557,7 @@ def YAMLCtor_CmdDefn(loader, node): fields['argdefns'] = fields.pop('arguments', None) return createCmdDefn(**fields) + def YAMLCtor_include(loader, node): # Get the path out of the yaml file name = os.path.join(os.path.dirname(loader.name), node.value) diff --git a/ait/core/newCmd.py b/ait/core/newCmd.py deleted file mode 100644 index ae2bfadc..00000000 --- a/ait/core/newCmd.py +++ /dev/null @@ -1,10 +0,0 @@ -from ait.core import cmd, log - - -class NewCmd(cmd.Cmd): - def encode(self): - log.info('We are in an extension of cmd.encode() that will then call the regular cmd.encode().') - return super(NewCmd, self).encode() - - def custom(self): - log.info("In a custom() method defined in the NewCmd extended class.") diff --git a/ait/core/newTable.py b/ait/core/newTable.py deleted file mode 100644 index a39c9b60..00000000 --- a/ait/core/newTable.py +++ /dev/null @@ -1,14 +0,0 @@ -from ait.core import table, log - - -class NewFSWTabDict(table.FSWTabDict): - def load(self): - log.info('Starting the table load() from the extended FSWTabDict class, using file: {}'.format(self.filename)) - return super(NewFSWTabDict, self).load() - - def create(self): - log.info('Starting the table create() from custom extension class') - return super(NewFSWTabDict, self).create() - - def custom(): - log.info("Test of a unique method defined in an extension.") diff --git a/ait/core/table.py b/ait/core/table.py index 7ab0ce74..5bb0d8e7 100644 --- a/ait/core/table.py +++ b/ait/core/table.py @@ -11,6 +11,7 @@ # laws and regulations. User has the responsibility to obtain export licenses, # or other export authority as may be required before exporting such # information to foreign countries or providing access to foreign persons. + import datetime import hashlib import io @@ -29,7 +30,8 @@ class FSWColDefn(object): """FSWColDefn - Argument Definition Argument Definitions encapsulate all information required to define - a single column. + a single column. + """ def __init__(self, *args, **kwargs): @@ -205,6 +207,7 @@ class FSWTabDefn(object): single column. This includes the column name, its opcode, subsystem, description and a list of argument definitions. Name and opcode are required. All others are optional. + """ def __init__(self, *args, **kwargs): @@ -373,6 +376,7 @@ def _decode_table_row(self, in_stream, raw=False): Raises: ValueError: When an EOFError is encountered while decoding any column but the first or if any column decode returns None. + """ row = [] @@ -404,6 +408,7 @@ class FSWTabDict(dict): Table Dictionaries provide a Python dictionary (i.e. hashtable) interface mapping Tables names to Column Definitions. + """ def __init__(self, *args, **kwargs): @@ -427,7 +432,7 @@ def create(self, name, *args): tab = None defn = self.get(name, None) if defn: - tab = FSWTab(defn, *args) + tab = createFSWTab(defn, *args) return tab def load(self, filename): @@ -461,7 +466,7 @@ def dirty(self): def load(self): if self.fswtabdict is None: if self.dirty(): - self.fswtabdict = FSWTabDict(self.filename) + self.fswtabdict = createFSWTabDict(self.filename) self.update() else: with open(self.pcklname, "rb") as stream: @@ -497,14 +502,14 @@ def getDefaultDict(): # noqa: N802 def YAMLCtor_FSWColDefn(loader, node): # noqa: N802 fields = loader.construct_mapping(node, deep=True) - return FSWColDefn(**fields) + return createFSWColDefn(**fields) def YAMLCtor_FSWTabDefn(loader, node): fields = loader.construct_mapping(node, deep=True) fields['fswheaderdefns'] = fields.pop('header', None) fields['coldefns'] = fields.pop('columns', None) - return FSWTabDefn(**fields) + return createFSWTabDefn(**fields) def encode_to_file(tbl_type, in_path, out_path): @@ -543,6 +548,4 @@ def decode_to_file(tbl_type, in_path, out_path): yaml.add_constructor("!FSWTable", YAMLCtor_FSWTabDefn) yaml.add_constructor("!FSWColumn", YAMLCtor_FSWColDefn) -print(f'__name__: {__name__}') - util.__init_extensions__(__name__, globals()) diff --git a/ait/core/util.py b/ait/core/util.py index 38601ef5..cbf3ebc4 100755 --- a/ait/core/util.py +++ b/ait/core/util.py @@ -36,57 +36,56 @@ class ObjectCache (object): def __init__(self, filename, loader): - """Creates a new ObjectCache + """ + Creates a new ObjectCache - Caches the Python object returned by loader(filename), using - Python's pickle object serialization mechanism. An ObjectCache - is useful when loader(filename) is slow. + Caches the Python object returned by loader(filename), using + Python's pickle object serialization mechanism. An ObjectCache + is useful when loader(filename) is slow. - The result of loader(filename) is cached to cachename, the - basename of filename with a '.pkl' extension. + The result of loader(filename) is cached to cachename, the + basename of filename with a '.pkl' extension. - Use the load() method to load, either via loader(filename) or - the pickled cache file, whichever was modified most recently. - """ - self._loader = loader - self._dict = None - self._filename = filename - self._cachename = os.path.splitext(filename)[0] + '.pkl' + Use the load() method to load, either via loader(filename) or + the pickled cache file, whichever was modified most recently. + """ + self._loader = loader + self._dict = None + self._filename = filename + self._cachename = os.path.splitext(filename)[0] + '.pkl' @property def cachename(self): - """The pickled cache filename""" - return self._cachename - + """The pickled cache filename""" + return self._cachename @property def dirty(self): - """True if the cache needs to be updated, False otherwise""" - return not os.path.exists(self.cachename) or \ - (os.path.getmtime(self.filename) > - os.path.getmtime(self.cachename)) - + """True if the cache needs to be updated, False otherwise""" + return not os.path.exists(self.cachename) or \ + (os.path.getmtime(self.filename) > + os.path.getmtime(self.cachename)) @property def filename(self): - """The filename to cache via loader(filename)""" - return self._filename - + """The filename to cache via loader(filename)""" + return self._filename def cache(self): - """Caches the result of loader(filename) to cachename.""" - msg = 'Saving updates from more recent "%s" to "%s"' - log.info(msg, self.filename, self.cachename) - with open(self.cachename, 'wb') as output: - pickle.dump(self._dict, output, -1) - + """Caches the result of loader(filename) to cachename.""" + msg = 'Saving updates from more recent "%s" to "%s"' + log.info(msg, self.filename, self.cachename) + with open(self.cachename, 'wb') as output: + pickle.dump(self._dict, output, -1) def load(self): - """Loads the Python object + """ + Loads the Python object Loads the Python object, either via loader(filename) or the pickled cache file, whichever was modified most recently. + """ if self._dict is None: if self.dirty: @@ -109,7 +108,8 @@ def load(self): def __init_extensions__(modname, modsyms): - """Initializes a module (given its name and :func:`globals()` symbol + """ + Initializes a module (given its name and :func:`globals()` symbol table) for AIT extensions. For every Python class defined in the given module, a @@ -128,10 +128,12 @@ def __init_extensions__(modname, modsyms): :func:`createCmd()`) it will now create a ``FooCmd`` object instead. Note: ``FooCmd`` should adhere to the same interface as :class:`ait.core.cmd.Cmd` (and probably inherit from it). + """ def createFunc (cls, extname): - """Creates and returns a new ``createXXX()`` function to instantiate + """ + Creates and returns a new ``createXXX()`` function to instantiate either the given class by class object (*cls*) or extension class name (*extname*). @@ -139,10 +141,10 @@ class name (*extname*). returned ``createXXX()`` is called, it attempts to lookup and load the class. Thereafter, the loaded class is cached for subsequent calls. + """ + def create(*args, **kwargs): - print("*****************") - print(create.cls) if create.cls is None: parts = extname.rsplit('.', 1) if len(parts) > 1: @@ -151,6 +153,7 @@ def create(*args, **kwargs): if module is None: raise ImportError('No module named %d' % modname) create.cls = getattr(module, clsname) + if create.cls is None: raise ImportError('No class named %s' % extname) return create.cls(*args, **kwargs) @@ -160,36 +163,32 @@ def create(*args, **kwargs): extensions = ait.config.get('extensions', None) for clsname, cls in modsyms.copy().items(): + if not isinstance(cls, type): continue extname = None - if extensions: extname = extensions.get(modname + '.' + clsname, None) - print(f"extname: {extname}") - if extname: cls = None values = modname, clsname, extname log.info('Replacing %s.%s with custom extension: %s' % values) - print('create' + clsname) - modsyms['create' + clsname] = createFunc(cls, extname) def __load_functions__ (symtbl): - """Loads all Python functions from the module specified in the + """ + Loads all Python functions from the module specified in the ``functions`` configuration parameter (in config.yaml) into the given symbol table (Python dictionary). + """ modname = ait.config.get('functions', None) - if modname: module = pydoc.locate(modname) - if module is None: msg = 'No module named %d (from config.yaml functions: parameter)' raise ImportError(msg % modname) @@ -201,13 +200,16 @@ def __load_functions__ (symtbl): def crc32File(filename, skip=0): - """Computes the CRC-32 of the contents of filename, optionally + """ + Computes the CRC-32 of the contents of filename, optionally skipping a certain number of bytes at the beginning of the file. + """ with open(filename, 'rb') as stream: discard = stream.read(skip) return zlib.crc32(stream.read()) & 0xffffffff + def endianSwapU16(bytes): """Swaps pairs of bytes (16-bit words) in the given bytearray.""" for b in range(0, len(bytes), 2): @@ -216,21 +218,25 @@ def endianSwapU16(bytes): def setDictDefaults (d, defaults): - """Sets all defaults for the given dictionary to those contained in a - second defaults dictionary. This convenience method calls: + """ + Sets all defaults for the given dictionary to those contained in a + second defaults dictionary. This convenience method calls: + + d.setdefault(key, value) - d.setdefault(key, value) + for each key and value in the given defaults dictionary. - for each key and value in the given defaults dictionary. - """ - for key, val in defaults.items(): - d.setdefault(key, val) + """ - return d + for key, val in defaults.items(): + d.setdefault(key, val) + + return d def getDefaultDict(modname, config_key, loader, reload=False, filename=None): - """Returns default AIT dictonary for modname + """ + Returns default AIT dictonary for modname This helper function encapulates the core logic necessary to (re)load, cache (via util.ObjectCache), and return the default @@ -238,6 +244,7 @@ def getDefaultDict(modname, config_key, loader, reload=False, filename=None): def getDefaultDict(reload=False): return ait.util.getDefaultDict(__name__, 'cmddict', CmdDict, reload) + """ module = sys.modules[modname] default = getattr(module, 'DefaultDict', None) @@ -278,7 +285,8 @@ def toBCD (n): def toFloat (str, default=None): - """toFloat(str[, default]) -> float | default + """ + toFloat(str[, default]) -> float | default Converts the given string to a floating-point value. If the string could not be converted, default (None) is returned. @@ -297,7 +305,9 @@ def toFloat (str, default=None): >>> f = toFloat("Foo") >>> assert f is None + """ + value = default try: @@ -309,7 +319,8 @@ def toFloat (str, default=None): def toNumber (str, default=None): - """toNumber(str[, default]) -> integer | float | default + """ + toNumber(str[, default]) -> integer | float | default Converts the given string to a numeric value. The string may be a hexadecimal, integer, or floating number. If string could not be @@ -331,7 +342,9 @@ def toNumber (str, default=None): >>> n = toNumber("Foo") >>> assert n is None + """ + value = default try: @@ -347,20 +360,28 @@ def toNumber (str, default=None): return value + def toNumberOrStr (str): - """toNumberOrStr(str) -> integer | float | string + """ + toNumberOrStr(str) -> integer | float | string Converts the given string to a numeric value, if possible. Otherwise returns the input string + """ + return toNumber(str, str) + def toRepr (obj): - """toRepr(obj) -> string + """ + toRepr(obj) -> string Converts the Python object to a string representation of the kind often returned by a class __repr__() method. + """ + args = [ ] names = [ ] @@ -382,8 +403,10 @@ def toRepr (obj): def toStringDuration (duration): - """Returns a description of the given duration in the most appropriate + """ + Returns a description of the given duration in the most appropriate units (e.g. seconds, ms, us, or ns). + """ table = ( @@ -401,10 +424,14 @@ def toStringDuration (duration): return '%fs' % duration + def expandPath (pathname, prefix=None): - """Return pathname as an absolute path, either expanded by the users + """ + Return pathname as an absolute path, either expanded by the users home directory ("~") or with prefix prepended. + """ + if prefix is None: prefix = '' @@ -417,10 +444,14 @@ def expandPath (pathname, prefix=None): return os.path.abspath(expanded) + def listAllFiles (directory, suffix=None, abspath=False): - """Returns the list of all files within the input directory and + """ + Returns the list of all files within the input directory and all subdirectories. + """ + files = [] directory = expandPath(directory) diff --git a/ait/extCmdTest.py b/ait/extCmdTest.py deleted file mode 100644 index 8b9d43f7..00000000 --- a/ait/extCmdTest.py +++ /dev/null @@ -1,7 +0,0 @@ -import ait.core.cmd as cmd -# import ait.core.log as log - -cmdDict = cmd.getDefaultDict() -no_op = cmdDict.create('NO_OP') -no_op.encode() -no_op.custom() diff --git a/ait/extTableTest.py b/ait/extTableTest.py deleted file mode 100644 index 18da3960..00000000 --- a/ait/extTableTest.py +++ /dev/null @@ -1,9 +0,0 @@ -import ait.core.table as table - -# tab_dict = table.FSWTabDict("/Users/jehofman/AIT-Core/config/table.yaml") -# tab_dict.load(filename=tab_dict.filename) - -table_dict = table.getDefaultFSWTabDict() -table_dict.create('MyTable') -table_dict.load(filename=table_dict.filename) - diff --git a/config/config.yaml b/config/config.yaml index 281e7c13..df9b3d8f 100755 --- a/config/config.yaml +++ b/config/config.yaml @@ -84,11 +84,6 @@ default: max_entity_id_length: 8 max_transaction_id_length: 8 - extensions: - ait.core.cmd.Cmd: ait.core.newCmd.NewCmd - ait.core.table.FSWTabDict: ait.core.newTable.NewFSWTabDict - - logging: name: ait hostname: yourCustomHostForLogging @@ -172,3 +167,7 @@ default: output: 3075 command_subscriber: True + + extensions: + ait.core.cmd.Cmd: ait.core.newCmd.NewCmd + ait.core.table.FSWTabDict: ait.core.newTable.NewFSWTabDict From 7def8d4ae4c485ced086e9434df27c53b7cd70d2 Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Thu, 24 Mar 2022 13:57:56 -0700 Subject: [PATCH 4/5] reverts in table, and update util --- ait/core/table.py | 8 ++++---- ait/core/util.py | 51 +++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/ait/core/table.py b/ait/core/table.py index 6b7cb758..735aef2a 100644 --- a/ait/core/table.py +++ b/ait/core/table.py @@ -433,7 +433,7 @@ def create(self, name, *args): tab = None defn = self.get(name, None) if defn: - tab = FSWTab(defn, *args) + tab = createFSWTab(defn, *args) return tab def load(self, filename): @@ -467,7 +467,7 @@ def dirty(self): def load(self): if self.fswtabdict is None: if self.dirty(): - self.fswtabdict = FSWTabDict(self.filename) + self.fswtabdict = createFSWTabDict(self.filename) self.update() else: with open(self.pcklname, "rb") as stream: @@ -503,14 +503,14 @@ def getDefaultDict(): # noqa: N802 def YAMLCtor_FSWColDefn(loader, node): # noqa: N802 fields = loader.construct_mapping(node, deep=True) - return FSWColDefn(**fields) + return createFSWColDefn(**fields) def YAMLCtor_FSWTabDefn(loader, node): # noqa: N802 fields = loader.construct_mapping(node, deep=True) fields["fswheaderdefns"] = fields.pop("header", None) fields["coldefns"] = fields.pop("columns", None) - return FSWTabDefn(**fields) + return createFSWTabDefn(**fields) def encode_to_file(tbl_type, in_path, out_path): diff --git a/ait/core/util.py b/ait/core/util.py index e99237bf..e0edb21a 100755 --- a/ait/core/util.py +++ b/ait/core/util.py @@ -59,10 +59,8 @@ def cachename(self): @property def dirty(self): - """True if the cache needs to be updated, False otherwise""" - return not os.path.exists(self.cachename) or \ - (os.path.getmtime(self.filename) > - os.path.getmtime(self.cachename)) + """True if the pickle cache needs to be updated, False to use pickle binary""" + return self.check_yaml_timestamps(self.filename, self.cachename) @property def filename(self): @@ -76,6 +74,51 @@ def cache(self): with open(self.cachename, "wb") as output: pickle.dump(self._dict, output, -1) + def check_yaml_timestamps(self, yaml_file_name, cache_name): + """ + Checks YAML configuration file timestamp and any 'included' YAML configuration file + timestamps against the pickle cache file. + The term 'dirty' means that a yaml file has a more recent timestamp than the pickle + cache file. If a file is found to be dirty the response will indicate that a new + pickle cache file must be generated. As soon as one file is found to be 'dirty' + the flag indicating 'dirty' will be returned. If a file_name is found to be 'clean' + the pickle binary will be loaded. + + param yaml_file_name: str + Name of the yaml configuration file to be tested + param cache_name: str + Filename with path to the cached pickle file for this config file. + + return: boolean + True/False indicating 'dirty' (update pickle cache) + + """ + + # If no pickle cache exists return True to make a new one. + if not os.path.exists(cache_name): + log.debug(f'No pickle cache exists, make a new one') + return True + # Has the yaml config file has been modified since the creation of the pickle cache + if os.path.getmtime(yaml_file_name) > os.path.getmtime(cache_name): + log.debug(f'{yaml_file_name} modified - make a new pickle cash') + return True + # Get the directory of the yaml config file to be parsed + dir_name = os.path.dirname(yaml_file_name) + # Open the yaml config file to look for '!includes' to be tested on the next iteration + with open(yaml_file_name, "r") as file: + try: + for line in file: + if not line.strip().startswith("#") and "!include" in line: + check = self.check_yaml_timestamps( + os.path.join(dir_name, line.strip().split(" ")[2]), cache_name) + if check: + return True + except RecursionError as e: # TODO Python 3.7 does not catch this error. + print(f'ERROR: {e}: Infinite loop: check that yaml config files are not looping ' + f'back and forth to one another thought the "!include" statements.') + log.debug('Load pickle binary.') + return False + def load(self): """ Loads the Python object From b0daf40d467faccd01d4625041625ff0293f7a9e Mon Sep 17 00:00:00 2001 From: Jim Hofman Date: Thu, 24 Mar 2022 13:59:59 -0700 Subject: [PATCH 5/5] conflict with a comment --- ait/core/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ait/core/util.py b/ait/core/util.py index e0edb21a..e3f8b498 100755 --- a/ait/core/util.py +++ b/ait/core/util.py @@ -113,7 +113,7 @@ def check_yaml_timestamps(self, yaml_file_name, cache_name): os.path.join(dir_name, line.strip().split(" ")[2]), cache_name) if check: return True - except RecursionError as e: # TODO Python 3.7 does not catch this error. + except RecursionError as e: # TODO Python 3.7 does not catch this error. print(f'ERROR: {e}: Infinite loop: check that yaml config files are not looping ' f'back and forth to one another thought the "!include" statements.') log.debug('Load pickle binary.')