From d9bd9fe192c21926cf1567bca5ce10b13e1ded51 Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 28 Feb 2020 12:06:40 -0800 Subject: [PATCH 01/27] Decode returns, NX support, some fixes --- Makefile | 20 ++- src/cisco_gnmi/client.py | 81 ++++++++++- src/cisco_gnmi/nx.py | 304 ++++++++++++++++++++++++++++++++++++++- src/cisco_gnmi/util.py | 1 + src/cisco_gnmi/xe.py | 30 ++-- 5 files changed, 417 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 4ffc67c..2d58e23 100644 --- a/Makefile +++ b/Makefile @@ -84,4 +84,22 @@ help: helpMessage = ""; \ } \ }' \ - $(MAKEFILE_LIST) \ No newline at end of file + $(MAKEFILE_LIST) + +## Setup links in virtual env for development +develop: + @echo "--------------------------------------------------------------------" + @echo "Setting up development environment" + @python setup.py develop --no-deps -q + @echo "" + @echo "Done." + @echo "" + +## Remove development links in virtual env +undevelop: + @echo "--------------------------------------------------------------------" + @echo "Removing development environment" + @python setup.py develop --no-deps -q --uninstall + @echo "" + @echo "Done." + @echo "" \ No newline at end of file diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index a46dcc0..574711d 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -24,6 +24,7 @@ """Python gNMI wrapper to ease usage of gNMI.""" import logging +from collections import OrderedDict from xml.etree.ElementPath import xpath_tokenizer_re from six import string_types @@ -152,7 +153,9 @@ def get( "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding ) request = proto.gnmi_pb2.GetRequest() - if not isinstance(paths, (list, set)): + try: + iter(paths) + except TypeError: raise Exception("paths must be an iterable containing Path(s)!") request.path.extend(paths) request.type = data_type @@ -334,3 +337,79 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): raise Exception("Unfinished elements in XPath parsing!") path.elem.extend(path_elems) return path + + def xpath_iterator(self, xpath): + for token in xpath[1:].split('/'): + #elem = proto.gnmi_pb2.PathElem() + xpath_so_far += '/' + token + if '[' in token: + keys = OrderedDict() + subtoken = token.replace('[', ',').replace(']', '').split(',') + for k in subtoken: + if '=' not in k: + elem = {'elem': OrderedDict({'name': k})} + #elem.name = k + else: + k, val = tuple(k.replace('"', '').split('=')) + keys['name'] = k + keys['value'] = val + elem['elem'].update({'key': keys}) + #elem.key.update(keys) + else: + elem = {'elem': {'name': token}} + #elem.name = token + yield elem + + def combine_segments(self, segments): + xpaths = [seg[0] for seg in segments] + prev_path = '' + extentions = [] + for path in xpaths: + if not prev_path: + prev_path = path + continue + if len(path) > len(prev_path): + short_path = prev_path + long_path = path + else: + short_path = path + long_path = prev_path + if short_path in long_path: + end = long_path[:len(short_path)].split() + extentions.append((short_path, end)) + removes = [seg for seg in segments if seg[0] == short_path] + for seg in removes: + segments.remove(seg) + import pdb; pdb.set_trace() + print(segments) + + + + def resolve_segments(self, segments, required_segments=[]): + duplicates = [] + if not segments: + return required_segments + xpath, elems, value = segments.pop(0) + for seg in segments: + if seg == (xpath, elems, value): + # Duplicate so move on + duplicates.append(seg) + continue + next_xpath, next_elems, next_value = seg + + if xpath in next_xpath: + # Check if segment is a key + for seg in segments: + #for elem in seg[1]: + if xpath != seg['elem'].get('keybase', ''): + continue + else: + break + else: + # This is a key + return self.resolve_segments( + segments, + required_segments + ) + required_segments.append((xpath, elems, value)) + return self.resolve_segments(segments, required_segments) diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 00f98f0..3746079 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -25,6 +25,8 @@ import logging +import json +import os from six import string_types from .client import Client, proto, util @@ -53,11 +55,302 @@ class NXClient(Client): >>> print(capabilities) """ - def get(self, *args, **kwargs): - raise NotImplementedError("Get not yet supported on NX-OS!") + def delete_xpaths(self, xpaths, prefix=None): + """A convenience wrapper for set() which constructs Paths from supplied xpaths + to be passed to set() as the delete parameter. - def set(self, *args, **kwargs): - raise NotImplementedError("Set not yet supported on NX-OS!") + Parameters + ---------- + xpaths : iterable of str + XPaths to specify to be deleted. + If prefix is specified these strings are assumed to be the suffixes. + prefix : str + The XPath prefix to apply to all XPaths for deletion. + + Returns + ------- + set() + """ + if isinstance(xpaths, string_types): + xpaths = [xpaths] + paths = [] + for xpath in xpaths: + if prefix: + if prefix.endswith("/") and xpath.startswith("/"): + xpath = "{prefix}{xpath}".format( + prefix=prefix[:-1], xpath=xpath[1:] + ) + elif prefix.endswith("/") or xpath.startswith("/"): + xpath = "{prefix}{xpath}".format(prefix=prefix, xpath=xpath) + else: + xpath = "{prefix}/{xpath}".format(prefix=prefix, xpath=xpath) + paths.append(self.parse_xpath_to_gnmi_path(xpath)) + return self.set(deletes=paths) + + def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): + """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. + All parameters are optional, but at least one must be present. + + This method expects JSON in the same format as what you might send via the native gRPC interface + with a fully modeled configuration which is then parsed to meet the gNMI implementation. + + Parameters + ---------- + update_json_configs : iterable of JSON configurations, optional + JSON configs to apply as updates. + replace_json_configs : iterable of JSON configurations, optional + JSON configs to apply as replacements. + ietf : bool, optional + Use JSON_IETF vs JSON. + + Returns + ------- + set() + """ + if not any([update_json_configs, replace_json_configs]): + raise Exception("Must supply at least one set of configurations to method!") + + def check_configs(configs): + if isinstance(configs, string_types): + logging.debug("Handling as JSON string.") + try: + configs = json.loads(configs) + except: + raise Exception("{0}\n is invalid JSON!".format(configs)) + configs = [configs] + elif isinstance(configs, dict): + logging.debug("Handling already serialized JSON object.") + configs = [configs] + elif not isinstance(configs, (list, set)): + raise Exception( + "{0} must be an iterable of configs!".format(str(configs)) + ) + return configs + + def segment_configs(configs=[]): + seg_config = [] + for config in configs: + top_element = next(iter(config.keys())) + name, val = (next(iter(config[top_element].items()))) + value = {'name': name, 'value': val} + seg_config.append( + ( + top_element, + [seg for seg in self.xpath_iterator(top_element)], + value + ) + ) + import pdb; pdb.set_trace() + seg_config = self.resolve_segments(seg_config) + # Build the Path + path = proto.gnmi_pb2.Path() + for config in seg_config: + xpath, segments, value = config + for seg in segments: + path_elem = proto.gnmi_pb2.PathElem() + path_elem.name = seg['elem']['name'] + return seg_config + + def create_updates(configs): + if not configs: + return None + configs = check_configs(configs) + import pdb; pdb.set_trace() + #segment_configs(configs) + updates = [] + for config in configs: + if not isinstance(config, dict): + raise Exception("config must be a JSON object!") + if len(config.keys()) > 1: + raise Exception("config should only target one YANG module!") + top_element = next(iter(config.keys())) + update = proto.gnmi_pb2.Update() + update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + config = config.pop(top_element) + if ietf: + update.val.json_ietf_val = json.dumps(config).encode("utf-8") + else: + update.val.json_val = json.dumps(config).encode("utf-8") + updates.append(update) + return updates + # def create_updates(configs): + # if not configs: + # return None + # configs = check_configs(configs) + # updates = [] + # xpaths = [] + # bottom_xpath = [] + # seg_keys = {} + # for config in configs: + # if not isinstance(config, dict): + # raise Exception("config must be a JSON object!") + # if len(config.keys()) > 1: + # raise Exception("config should only target one YANG module!") + # xpaths.append(next(iter(config.keys()))) + # top = os.path.dirname(os.path.commonprefix(xpaths)) + # top_xpath = [s['segment'] for s in self.xpath_iterator(top)] + # bottom_xpath = [s for s in self.xpath_iterator(xpath[len(top):]) + # for xpath in xpaths: + # for seg in self.xpath_iterator(xpath[len(top):]): + # if 'keys' in seg: + # if not seg_keys: + # seg_keys = seg['keys'] + # else: + # for k,v in seg['keys'].items(): + # if k in seg_keys: + # seg_keys[k].update(v) + # else: + # seg_keys[k] = v + # else: + # bottom_xpath.append(seg['segment']) + # for seg in bottom_xpath: + # if seg not in top_xpath: + # top_xpath.append(seg) + # for key, val in seg_keys.items(): + # top_xpath[top_xpath.index(key)] = {key: val} + # import pdb; pdb.set_trace() + + updates = create_updates(update_json_configs) + replaces = create_updates(replace_json_configs) + return self.set(updates=updates, replaces=replaces) + + def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON"): + """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. + + Parameters + ---------- + xpaths : iterable of str or str + An iterable of XPath strings to request data of + If simply a str, wraps as a list for convenience + data_type : proto.gnmi_pb2.GetRequest.DataType, optional + A direct value or key from the GetRequest.DataType enum + [ALL, CONFIG, STATE, OPERATIONAL] + encoding : proto.gnmi_pb2.GetRequest.Encoding, optional + A direct value or key from the Encoding enum + [JSON, JSON_IETF] + + Returns + ------- + get() + """ + supported_encodings = ["JSON"] + encoding = util.validate_proto_enum( + "encoding", + encoding, + "Encoding", + proto.gnmi_pb2.Encoding, + supported_encodings, + ) + gnmi_path = None + if isinstance(xpaths, (list, set)): + gnmi_path = map(self.parse_xpath_to_gnmi_path, set(xpaths)) + elif isinstance(xpaths, string_types): + gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths)] + else: + raise Exception( + "xpaths must be a single xpath string or iterable of xpath strings!" + ) + return self.get(gnmi_path, data_type=data_type, encoding=encoding) + + def subscribe_xpaths( + self, + xpath_subscriptions, + encoding="JSON_IETF", + sample_interval=Client._NS_IN_S * 10, + heartbeat_interval=None, + ): + """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest + with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, + dictionaries with Subscription attributes for more granularity, or already built Subscription + objects and builds the SubscriptionList. Fields not supplied will be defaulted with the default arguments + to the method. + + Generates a single SubscribeRequest. + + Parameters + ---------- + xpath_subscriptions : str or iterable of str, dict, Subscription + An iterable which is parsed to form the Subscriptions in the SubscriptionList to be passed + to SubscriptionRequest. Strings are parsed as XPaths and defaulted with the default arguments, + dictionaries are treated as dicts of args to pass to the Subscribe init, and Subscription is + treated as simply a pre-made Subscription. + encoding : proto.gnmi_pb2.Encoding, optional + A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data + [JSON, JSON_IETF] + sample_interval : int, optional + Default nanoseconds for sample to occur. + Defaults to 10 seconds. + heartbeat_interval : int, optional + Specifies the maximum allowable silent period in nanoseconds when + suppress_redundant is in use. The target should send a value at least once + in the period specified. + + Returns + ------- + subscribe() + """ + supported_request_modes = ["STREAM"] + request_mode = "STREAM" + supported_sub_modes = ["SAMPLE"] + sub_mode = "SAMPLE" + supported_encodings = ["JSON", "JSON_IETF"] + subscription_list = proto.gnmi_pb2.SubscriptionList() + subscription_list.mode = util.validate_proto_enum( + "mode", + request_mode, + "SubscriptionList.Mode", + proto.gnmi_pb2.SubscriptionList.Mode, + supported_request_modes, + ) + subscription_list.encoding = util.validate_proto_enum( + "encoding", + encoding, + "Encoding", + proto.gnmi_pb2.Encoding, + supported_encodings, + ) + if isinstance(xpath_subscriptions, string_types): + xpath_subscriptions = [xpath_subscriptions] + subscriptions = [] + for xpath_subscription in xpath_subscriptions: + subscription = None + if isinstance(xpath_subscription, string_types): + subscription = proto.gnmi_pb2.Subscription() + subscription.path.CopyFrom( + self.parse_xpath_to_gnmi_path(xpath_subscription) + ) + subscription.mode = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + supported_sub_modes, + ) + subscription.sample_interval = sample_interval + elif isinstance(xpath_subscription, dict): + path = self.parse_xpath_to_gnmi_path(xpath_subscription["path"]) + arg_dict = { + "path": path, + "mode": sub_mode, + "sample_interval": sample_interval, + } + arg_dict.update(xpath_subscription) + if "mode" in arg_dict: + arg_dict["mode"] = util.validate_proto_enum( + "sub_mode", + arg_dict["mode"], + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + supported_sub_modes, + ) + subscription = proto.gnmi_pb2.Subscription(**arg_dict) + elif isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): + subscription = xpath_subscription + else: + raise Exception("xpath in list must be xpath or dict/Path!") + subscriptions.append(subscription) + subscription_list.subscription.extend(subscriptions) + return self.subscribe([subscription_list]) def subscribe_xpaths( self, @@ -173,8 +466,9 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): "OpenConfig data models not yet supported on NX-OS!" ) if origin is None: - if any(map(xpath.startswith, ["Cisco-NX-OS-device", "ietf-interfaces"])): + if any(map(xpath.startswith, ["/Cisco-NX-OS-device", "/ietf-interfaces"])): origin = "device" else: origin = "DME" + origin=None return super(NXClient, self).parse_xpath_to_gnmi_path(xpath, origin) diff --git a/src/cisco_gnmi/util.py b/src/cisco_gnmi/util.py index f65f104..bca78a7 100644 --- a/src/cisco_gnmi/util.py +++ b/src/cisco_gnmi/util.py @@ -110,6 +110,7 @@ def get_cn_from_cert(cert_pem): Defaults to first found if multiple CNs identified. """ cert_cn = None + import pdb; pdb.set_trace() cert_parsed = x509.load_pem_x509_certificate(cert_pem, default_backend()) cert_cns = cert_parsed.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME) if len(cert_cns) > 0: diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 91e1221..6e17e7b 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -131,27 +131,27 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru if not any([update_json_configs, replace_json_configs]): raise Exception("Must supply at least one set of configurations to method!") - def check_configs(name, configs): - if isinstance(name, string_types): - logging.debug("Handling %s as JSON string.", name) + def check_configs(configs): + if isinstance(configs, string_types): + logging.debug("Handling as JSON string.") try: configs = json.loads(configs) except: - raise Exception("{name} is invalid JSON!".format(name=name)) + raise Exception("{0}\n is invalid JSON!".format(configs)) configs = [configs] - elif isinstance(name, dict): - logging.debug("Handling %s as already serialized JSON object.", name) + elif isinstance(configs, dict): + logging.debug("Handling already serialized JSON object.") configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( - "{name} must be an iterable of configs!".format(name=name) + "{0} must be an iterable of configs!".format(str(configs)) ) return configs - def create_updates(name, configs): + def create_updates(configs): if not configs: return None - configs = check_configs(name, configs) + configs = check_configs(configs) updates = [] for config in configs: if not isinstance(config, dict): @@ -159,6 +159,12 @@ def create_updates(name, configs): if len(config.keys()) > 1: raise Exception("config should only target one YANG module!") top_element = next(iter(config.keys())) + # start mike + # path_obj = self.parse_xpath_to_gnmi_path(top_element) + # config = config.pop(top_element) + # value_obj = proto.gnmi_pb2.TypedValue(json_ietf_val=json.dumps(config).encode("utf-8")) + # update = proto.gnmi_pb2.Update(path=path_obj, val=value_obj) + # end mike update = proto.gnmi_pb2.Update() update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) config = config.pop(top_element) @@ -169,8 +175,8 @@ def create_updates(name, configs): updates.append(update) return updates - updates = create_updates("update_json_configs", update_json_configs) - replaces = create_updates("replace_json_configs", replace_json_configs) + updates = create_updates(update_json_configs) + replaces = create_updates(replace_json_configs) return self.set(updates=updates, replaces=replaces) def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF"): @@ -318,7 +324,7 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): """ if origin is None: # naive but effective - if ":" not in xpath: + if "openconfig" in xpath: origin = "openconfig" else: origin = "rfc7951" From 294a48f1cc510d2dd00ba78636eb20b50bc22ee8 Mon Sep 17 00:00:00 2001 From: miott Date: Thu, 5 Mar 2020 08:26:48 -0800 Subject: [PATCH 02/27] Added make develop, checkpoint commit for the rest --- Makefile | 4 +- src/cisco_gnmi/client.py | 1 - src/cisco_gnmi/nx.py | 221 +++++++++++++++++---------------------- src/cisco_gnmi/xe.py | 70 +++++++++++++ 4 files changed, 168 insertions(+), 128 deletions(-) diff --git a/Makefile b/Makefile index 2d58e23..47611af 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ help: develop: @echo "--------------------------------------------------------------------" @echo "Setting up development environment" - @python setup.py develop --no-deps -q + @python setup.py develop -q @echo "" @echo "Done." @echo "" @@ -99,7 +99,7 @@ develop: undevelop: @echo "--------------------------------------------------------------------" @echo "Removing development environment" - @python setup.py develop --no-deps -q --uninstall + @python setup.py develop -q --uninstall @echo "" @echo "Done." @echo "" \ No newline at end of file diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 574711d..c3d79b3 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -341,7 +341,6 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): def xpath_iterator(self, xpath): for token in xpath[1:].split('/'): #elem = proto.gnmi_pb2.PathElem() - xpath_so_far += '/' + token if '[' in token: keys = OrderedDict() subtoken = token.replace('[', ',').replace(']', '').split(',') diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 3746079..6b1323d 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -55,6 +55,76 @@ class NXClient(Client): >>> print(capabilities) """ + def xpath_to_path_elem(self, request): + paths = [] + message = { + 'update': [], + 'replace': [], + 'delete': [], + 'get': [], + } + if 'nodes' not in request: + # TODO: raw rpc? + return paths + else: + namespace_modules = {} + for prefix, nspace in request.get('namespace', {}).items(): + module = '' + if '/Cisco-IOS-' in nspace: + module = nspace[nspace.rfind('/') + 1:] + elif '/cisco-nx' in nspace: # NXOS lowercases namespace + module = 'Cisco-NX-OS-device' + elif '/openconfig.net' in nspace: + module = 'openconfig-' + module += nspace[nspace.rfind('/') + 1:] + elif 'urn:ietf:params:xml:ns:yang:' in nspace: + module = nspace.replace( + 'urn:ietf:params:xml:ns:yang:', '') + if module: + namespace_modules[prefix] = module + for node in request.get('nodes', []): + if 'xpath' not in node: + log.error('Xpath is not in message') + else: + xpath = node['xpath'] + value = node.get('value', '') + edit_op = node.get('edit-op', '') + + for pfx, ns in namespace_modules.items(): + xpath = xpath.replace(pfx + ':', '') + value = value.replace(pfx + ':', '') + if edit_op: + if edit_op in ['create', 'merge', 'replace']: + xpath_lst = xpath.split('/') + name = xpath_lst.pop() + xpath = '/'.join(xpath_lst) + if edit_op == 'replace': + if not message['replace']: + message['replace'] = [{ + xpath: {name: value} + }] + else: + message['replace'].append( + {xpath: {name: value}} + ) + else: + if not message['update']: + message['update'] = [{ + xpath: {name: value} + }] + else: + message['update'].append( + {xpath: {name: value}} + ) + elif edit_op in ['delete', 'remove']: + if message['delete']: + message['delete'].add(xpath) + else: + message['delete'] = set(xpath) + else: + message['get'].append(xpath) + return namespace_modules, message + def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. @@ -87,6 +157,31 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) + def segment_configs(self, request, configs=[]): + seg_config = [] + for config in configs: + top_element = next(iter(config.keys())) + name, val = (next(iter(config[top_element].items()))) + value = {'name': name, 'value': val} + seg_config.append( + ( + top_element, + [seg for seg in self.xpath_iterator(top_element)], + value + ) + ) + import pdb; pdb.set_trace() + # seg_config = self.resolve_segments(seg_config) + # Build the Path + path = proto.gnmi_pb2.Path() + for config in seg_config: + xpath, segments, value = config + for seg in segments: + path_elem = proto.gnmi_pb2.PathElem() + path_elem.name = seg['elem']['name'] + + return seg_config + def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -127,36 +222,12 @@ def check_configs(configs): ) return configs - def segment_configs(configs=[]): - seg_config = [] - for config in configs: - top_element = next(iter(config.keys())) - name, val = (next(iter(config[top_element].items()))) - value = {'name': name, 'value': val} - seg_config.append( - ( - top_element, - [seg for seg in self.xpath_iterator(top_element)], - value - ) - ) - import pdb; pdb.set_trace() - seg_config = self.resolve_segments(seg_config) - # Build the Path - path = proto.gnmi_pb2.Path() - for config in seg_config: - xpath, segments, value = config - for seg in segments: - path_elem = proto.gnmi_pb2.PathElem() - path_elem.name = seg['elem']['name'] - return seg_config - def create_updates(configs): if not configs: return None configs = check_configs(configs) import pdb; pdb.set_trace() - #segment_configs(configs) + #self.segment_configs(configs) updates = [] for config in configs: if not isinstance(config, dict): @@ -252,106 +323,6 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON"): ) return self.get(gnmi_path, data_type=data_type, encoding=encoding) - def subscribe_xpaths( - self, - xpath_subscriptions, - encoding="JSON_IETF", - sample_interval=Client._NS_IN_S * 10, - heartbeat_interval=None, - ): - """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest - with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, - dictionaries with Subscription attributes for more granularity, or already built Subscription - objects and builds the SubscriptionList. Fields not supplied will be defaulted with the default arguments - to the method. - - Generates a single SubscribeRequest. - - Parameters - ---------- - xpath_subscriptions : str or iterable of str, dict, Subscription - An iterable which is parsed to form the Subscriptions in the SubscriptionList to be passed - to SubscriptionRequest. Strings are parsed as XPaths and defaulted with the default arguments, - dictionaries are treated as dicts of args to pass to the Subscribe init, and Subscription is - treated as simply a pre-made Subscription. - encoding : proto.gnmi_pb2.Encoding, optional - A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data - [JSON, JSON_IETF] - sample_interval : int, optional - Default nanoseconds for sample to occur. - Defaults to 10 seconds. - heartbeat_interval : int, optional - Specifies the maximum allowable silent period in nanoseconds when - suppress_redundant is in use. The target should send a value at least once - in the period specified. - - Returns - ------- - subscribe() - """ - supported_request_modes = ["STREAM"] - request_mode = "STREAM" - supported_sub_modes = ["SAMPLE"] - sub_mode = "SAMPLE" - supported_encodings = ["JSON", "JSON_IETF"] - subscription_list = proto.gnmi_pb2.SubscriptionList() - subscription_list.mode = util.validate_proto_enum( - "mode", - request_mode, - "SubscriptionList.Mode", - proto.gnmi_pb2.SubscriptionList.Mode, - supported_request_modes, - ) - subscription_list.encoding = util.validate_proto_enum( - "encoding", - encoding, - "Encoding", - proto.gnmi_pb2.Encoding, - supported_encodings, - ) - if isinstance(xpath_subscriptions, string_types): - xpath_subscriptions = [xpath_subscriptions] - subscriptions = [] - for xpath_subscription in xpath_subscriptions: - subscription = None - if isinstance(xpath_subscription, string_types): - subscription = proto.gnmi_pb2.Subscription() - subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path(xpath_subscription) - ) - subscription.mode = util.validate_proto_enum( - "sub_mode", - sub_mode, - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription.sample_interval = sample_interval - elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path(xpath_subscription["path"]) - arg_dict = { - "path": path, - "mode": sub_mode, - "sample_interval": sample_interval, - } - arg_dict.update(xpath_subscription) - if "mode" in arg_dict: - arg_dict["mode"] = util.validate_proto_enum( - "sub_mode", - arg_dict["mode"], - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription = proto.gnmi_pb2.Subscription(**arg_dict) - elif isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): - subscription = xpath_subscription - else: - raise Exception("xpath in list must be xpath or dict/Path!") - subscriptions.append(subscription) - subscription_list.subscription.extend(subscriptions) - return self.subscribe([subscription_list]) - def subscribe_xpaths( self, xpath_subscriptions, diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 6e17e7b..f39a48b 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -76,6 +76,76 @@ class XEClient(Client): >>> delete_response = client.delete_xpaths('/Cisco-IOS-XE-native:native/hostname') """ + def xpath_to_path_elem(self, request): + paths = [] + message = { + 'update': [], + 'replace': [], + 'delete': [], + 'get': [], + } + if 'nodes' not in request: + # TODO: raw rpc? + return paths + else: + namespace_modules = {} + for prefix, nspace in request.get('namespace', {}).items(): + module = '' + if '/Cisco-IOS-' in nspace: + module = nspace[nspace.rfind('/') + 1:] + elif '/cisco-nx' in nspace: # NXOS lowercases namespace + module = 'Cisco-NX-OS-device' + elif '/openconfig.net' in nspace: + module = 'openconfig-' + module += nspace[nspace.rfind('/') + 1:] + elif 'urn:ietf:params:xml:ns:yang:' in nspace: + module = nspace.replace( + 'urn:ietf:params:xml:ns:yang:', '') + if module: + namespace_modules[prefix] = module + for node in request.get('nodes', []): + if 'xpath' not in node: + log.error('Xpath is not in message') + else: + xpath = node['xpath'] + value = node.get('value', '') + edit_op = node.get('edit-op', '') + + for pfx, ns in namespace_modules.items(): + xpath = xpath.replace(pfx + ':', ns + ':') + value = value.replace(pfx + ':', ns + ':') + if edit_op: + if edit_op in ['create', 'merge', 'replace']: + xpath_lst = xpath.split('/') + name = xpath_lst.pop() + xpath = '/'.join(xpath_lst) + if edit_op == 'replace': + if not message['replace']: + message['replace'] = [{ + xpath: {name: value} + }] + else: + message['replace'].append( + {xpath: {name: value}} + ) + else: + if not message['update']: + message['update'] = [{ + xpath: {name: value} + }] + else: + message['update'].append( + {xpath: {name: value}} + ) + elif edit_op in ['delete', 'remove']: + if message['delete']: + message['delete'].add(xpath) + else: + message['delete'] = set(xpath) + else: + message['get'].append(xpath) + return namespace_modules, message + def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. From 1fd25abfafb0c921e46e2e1d3290b110f6112fb3 Mon Sep 17 00:00:00 2001 From: miott Date: Sun, 15 Mar 2020 14:16:52 -0700 Subject: [PATCH 03/27] Complex set working. --- src/cisco_gnmi/client.py | 239 +++++++++++++++++++++++++++------------ src/cisco_gnmi/nx.py | 207 ++++++++++++++------------------- 2 files changed, 256 insertions(+), 190 deletions(-) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index c3d79b3..ddae60c 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -26,6 +26,9 @@ import logging from collections import OrderedDict from xml.etree.ElementPath import xpath_tokenizer_re +import re +import json +import os from six import string_types from . import proto @@ -338,77 +341,171 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): path.elem.extend(path_elems) return path - def xpath_iterator(self, xpath): - for token in xpath[1:].split('/'): - #elem = proto.gnmi_pb2.PathElem() - if '[' in token: - keys = OrderedDict() - subtoken = token.replace('[', ',').replace(']', '').split(',') - for k in subtoken: - if '=' not in k: - elem = {'elem': OrderedDict({'name': k})} - #elem.name = k + def combine_configs(self, payload, last_xpath, xpath, config): + last_set = set(last_xpath.split('/')) + curr_diff = set(xpath.split('/')) - last_set + if len(curr_diff) > 1: + print('combine_configs() error1') + return payload + index = curr_diff.pop() + curr_xpath = xpath[xpath.find(index):] + curr_xpath = curr_xpath.split('/') + curr_xpath.reverse() + for seg in curr_xpath: + config = {seg: config} + + last_diff = last_set - set(xpath.split('/')) + if len(last_diff) > 1: + print('combine_configs() error2') + return payload + last_xpath = last_xpath[last_xpath.find(last_diff.pop()):] + last_xpath = last_xpath.split('/') + last_xpath.reverse() + for seg in last_xpath: + if seg not in payload: + payload = {seg: payload} + payload.update(config) + return payload + + def xpath_to_json(self, configs, last_xpath='', payload={}): + for i, cfg in enumerate(configs, 1): + xpath, config, is_key = cfg + if last_xpath and xpath not in last_xpath: + # Branched config here + # |---last xpath config + # --| + # |---this xpath config + payload = self.combine_configs(payload, last_xpath, xpath, config) + return self.xpath_to_json(configs[i:], xpath, payload) + xpath_segs = xpath.split('/') + xpath_segs.reverse() + for seg in xpath_segs: + if not seg: + continue + if payload: + if is_key: + if seg in payload: + if isinstance(payload[seg], list): + payload[seg].append(config) + elif isinstance(payload[seg], dict): + payload[seg].update(config) + else: + payload.update(config) + payload = {seg: [payload]} else: - k, val = tuple(k.replace('"', '').split('=')) - keys['name'] = k - keys['value'] = val - elem['elem'].update({'key': keys}) - #elem.key.update(keys) - else: - elem = {'elem': {'name': token}} - #elem.name = token - yield elem - - def combine_segments(self, segments): - xpaths = [seg[0] for seg in segments] - prev_path = '' - extentions = [] - for path in xpaths: - if not prev_path: - prev_path = path - continue - if len(path) > len(prev_path): - short_path = prev_path - long_path = path - else: - short_path = path - long_path = prev_path - if short_path in long_path: - end = long_path[:len(short_path)].split() - extentions.append((short_path, end)) - removes = [seg for seg in segments if seg[0] == short_path] - for seg in removes: - segments.remove(seg) - import pdb; pdb.set_trace() - print(segments) - - - - def resolve_segments(self, segments, required_segments=[]): - duplicates = [] - if not segments: - return required_segments - xpath, elems, value = segments.pop(0) - for seg in segments: - if seg == (xpath, elems, value): - # Duplicate so move on - duplicates.append(seg) - continue - next_xpath, next_elems, next_value = seg - - if xpath in next_xpath: - # Check if segment is a key - for seg in segments: - #for elem in seg[1]: - if xpath != seg['elem'].get('keybase', ''): - continue + config.update(payload) + payload = {seg: config} + return self.xpath_to_json(configs[i:], xpath, payload) + else: + if is_key: + payload = {seg: [config]} else: - break + payload = {seg: config} + return self.xpath_to_json(configs[i:], xpath, payload) + return payload + + # Pattern to detect keys in an xpath + RE_FIND_KEYS = re.compile(r'\[.*?\]') + + def get_payload(self, configs): + # Number of updates are limited so try to consolidate into lists. + xpaths_cfg = [] + first_key = set() + # Find first common keys for all xpaths_cfg of collection. + for config in configs: + xpath = next(iter(config.keys())) + + # Change configs to tuples (xpath, config) for easier management + xpaths_cfg.append((xpath, config[xpath])) + + xpath_split = xpath.split('/') + for seg in xpath_split: + if '[' in seg: + first_key.add(seg) + break + + # Common first key/configs represents one GNMI update + updates = [] + for key in first_key: + update = [] + remove_cfg = [] + for config in xpaths_cfg: + xpath, cfg = config + if key in xpath: + update.append(config) else: - # This is a key - return self.resolve_segments( - segments, - required_segments - ) - required_segments.append((xpath, elems, value)) - return self.resolve_segments(segments, required_segments) + for k, v in cfg.items(): + if '[{0}="{1}"]'.format(k, v) not in key: + break + else: + # This cfg sets the first key so we don't need it + remove_cfg.append((xpath, cfg)) + if update: + for upd in update: + # Remove this config out of main list + xpaths_cfg.remove(upd) + for rem_cfg in remove_cfg: + # Sets a key in update path so remove it + xpaths_cfg.remove(rem_cfg) + updates.append(update) + break + + # Add remaining configs to updates + if xpaths_cfg: + updates.append(xpaths_cfg) + + # Combine all xpath configs of each update if possible + xpaths = [] + compressed_updates = [] + for update in updates: + xpath_consolidated = {} + config_compressed = [] + for seg in update: + xpath, config = seg + if xpath in xpath_consolidated: + xpath_consolidated[xpath].update(config) + else: + xpath_consolidated[xpath] = config + config_compressed.append((xpath, xpath_consolidated[xpath])) + xpaths.append(xpath) + + # Now get the update path for this batch of configs + common_xpath = os.path.commonprefix(xpaths) + cfg_compressed = [] + keys = [] + + # Need to reverse the configs to build the dict correctly + config_compressed.reverse() + for seg in config_compressed: + is_key = False + prepend_path = '' + xpath, config = seg + end_path = xpath[len(common_xpath):] + if end_path.startswith('['): + # Don't start payload with a list + tmp = common_xpath.split('/') + prepend_path = '/' + tmp.pop() + common_xpath = '/'.join(tmp) + end_path = prepend_path + end_path + + # Building json, need to identify configs that set keys + for key in keys: + if [k for k in config.keys() if k in key]: + is_key = True + keys += re.findall(self.RE_FIND_KEYS, end_path) + cfg_compressed.append((end_path, config, is_key)) + + update = (common_xpath, cfg_compressed) + compressed_updates.append(update) + + updates = [] + for update in compressed_updates: + common_xpath, cfgs = update + payload = self.xpath_to_json(cfgs) + updates.append( + ( + common_xpath, + json.dumps(payload).encode('utf-8') + ) + ) + return updates diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 6b1323d..8476e39 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -68,8 +68,8 @@ def xpath_to_path_elem(self, request): return paths else: namespace_modules = {} + origin = 'DME' for prefix, nspace in request.get('namespace', {}).items(): - module = '' if '/Cisco-IOS-' in nspace: module = nspace[nspace.rfind('/') + 1:] elif '/cisco-nx' in nspace: # NXOS lowercases namespace @@ -82,6 +82,7 @@ def xpath_to_path_elem(self, request): 'urn:ietf:params:xml:ns:yang:', '') if module: namespace_modules[prefix] = module + for node in request.get('nodes', []): if 'xpath' not in node: log.error('Xpath is not in message') @@ -91,8 +92,15 @@ def xpath_to_path_elem(self, request): edit_op = node.get('edit-op', '') for pfx, ns in namespace_modules.items(): - xpath = xpath.replace(pfx + ':', '') - value = value.replace(pfx + ':', '') + # NXOS does not support prefixes yet so clear them out + if pfx in xpath and 'openconfig' in ns: + origin = 'openconfig' + xpath = xpath.replace(pfx + ':', '') + value = value.replace(pfx + ':', '') + elif pfx in xpath and 'device' in ns: + origin = 'device' + xpath = xpath.replace(pfx + ':', '') + value = value.replace(pfx + ':', '') if edit_op: if edit_op in ['create', 'merge', 'replace']: xpath_lst = xpath.split('/') @@ -123,7 +131,7 @@ def xpath_to_path_elem(self, request): message['delete'] = set(xpath) else: message['get'].append(xpath) - return namespace_modules, message + return namespace_modules, message, origin def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths @@ -157,32 +165,64 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def segment_configs(self, request, configs=[]): - seg_config = [] + def check_configs(self, configs): + if isinstance(configs, string_types): + logging.debug("Handling as JSON string.") + try: + configs = json.loads(configs) + except: + raise Exception("{0}\n is invalid JSON!".format(configs)) + configs = [configs] + elif isinstance(configs, dict): + logging.debug("Handling already serialized JSON object.") + configs = [configs] + elif not isinstance(configs, (list, set)): + raise Exception( + "{0} must be an iterable of configs!".format(str(configs)) + ) + return configs + + def create_updates(self, configs, origin): + if not configs: + return None + configs = self.check_configs(configs) + + xpaths = [] + updates = [] for config in configs: - top_element = next(iter(config.keys())) - name, val = (next(iter(config[top_element].items()))) - value = {'name': name, 'value': val} - seg_config.append( - ( - top_element, - [seg for seg in self.xpath_iterator(top_element)], - value + xpath = next(iter(config.keys())) + xpaths.append(xpath) + common_xpath = os.path.commonprefix(xpaths) + + if common_xpath: + update_configs = self.get_payload(configs) + for update_cfg in update_configs: + xpath, payload = update_cfg + update = proto.gnmi_pb2.Update() + update.path.CopyFrom( + self.parse_xpath_to_gnmi_path( + xpath, origin=origin + ) ) - ) - import pdb; pdb.set_trace() - # seg_config = self.resolve_segments(seg_config) - # Build the Path - path = proto.gnmi_pb2.Path() - for config in seg_config: - xpath, segments, value = config - for seg in segments: - path_elem = proto.gnmi_pb2.PathElem() - path_elem.name = seg['elem']['name'] - - return seg_config - - def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): + update.val.json_val = payload + updates.append(update) + logging.info('GNMI set:\n\n{0}'.format(str(update))) + return updates + else: + for config in configs: + top_element = next(iter(config.keys())) + update = proto.gnmi_pb2.Update() + update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + config = config.pop(top_element) + if ietf: + update.val.json_ietf_val = json.dumps(config).encode("utf-8") + else: + update.val.json_val = json.dumps(config).encode("utf-8") + updates.append(update) + return updates + + def set_json(self, update_json_configs=None, replace_json_configs=None, + origin='device'): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -195,8 +235,7 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru JSON configs to apply as updates. replace_json_configs : iterable of JSON configurations, optional JSON configs to apply as replacements. - ietf : bool, optional - Use JSON_IETF vs JSON. + origin : openconfig, device, or DME Returns ------- @@ -205,87 +244,12 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru if not any([update_json_configs, replace_json_configs]): raise Exception("Must supply at least one set of configurations to method!") - def check_configs(configs): - if isinstance(configs, string_types): - logging.debug("Handling as JSON string.") - try: - configs = json.loads(configs) - except: - raise Exception("{0}\n is invalid JSON!".format(configs)) - configs = [configs] - elif isinstance(configs, dict): - logging.debug("Handling already serialized JSON object.") - configs = [configs] - elif not isinstance(configs, (list, set)): - raise Exception( - "{0} must be an iterable of configs!".format(str(configs)) - ) - return configs - - def create_updates(configs): - if not configs: - return None - configs = check_configs(configs) - import pdb; pdb.set_trace() - #self.segment_configs(configs) - updates = [] - for config in configs: - if not isinstance(config, dict): - raise Exception("config must be a JSON object!") - if len(config.keys()) > 1: - raise Exception("config should only target one YANG module!") - top_element = next(iter(config.keys())) - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) - config = config.pop(top_element) - if ietf: - update.val.json_ietf_val = json.dumps(config).encode("utf-8") - else: - update.val.json_val = json.dumps(config).encode("utf-8") - updates.append(update) - return updates - # def create_updates(configs): - # if not configs: - # return None - # configs = check_configs(configs) - # updates = [] - # xpaths = [] - # bottom_xpath = [] - # seg_keys = {} - # for config in configs: - # if not isinstance(config, dict): - # raise Exception("config must be a JSON object!") - # if len(config.keys()) > 1: - # raise Exception("config should only target one YANG module!") - # xpaths.append(next(iter(config.keys()))) - # top = os.path.dirname(os.path.commonprefix(xpaths)) - # top_xpath = [s['segment'] for s in self.xpath_iterator(top)] - # bottom_xpath = [s for s in self.xpath_iterator(xpath[len(top):]) - # for xpath in xpaths: - # for seg in self.xpath_iterator(xpath[len(top):]): - # if 'keys' in seg: - # if not seg_keys: - # seg_keys = seg['keys'] - # else: - # for k,v in seg['keys'].items(): - # if k in seg_keys: - # seg_keys[k].update(v) - # else: - # seg_keys[k] = v - # else: - # bottom_xpath.append(seg['segment']) - # for seg in bottom_xpath: - # if seg not in top_xpath: - # top_xpath.append(seg) - # for key, val in seg_keys.items(): - # top_xpath[top_xpath.index(key)] = {key: val} - # import pdb; pdb.set_trace() - - updates = create_updates(update_json_configs) - replaces = create_updates(replace_json_configs) + updates = self.create_updates(update_json_configs, origin=origin) + replaces = self.create_updates(replace_json_configs, origin=origin) return self.set(updates=updates, replaces=replaces) - def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON"): + def get_xpaths(self, xpaths, data_type="ALL", + encoding="JSON", origin='openconfig'): """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. Parameters @@ -314,9 +278,11 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON"): ) gnmi_path = None if isinstance(xpaths, (list, set)): - gnmi_path = map(self.parse_xpath_to_gnmi_path, set(xpaths)) + gnmi_path = [] + for xpath in set(xpaths): + gnmi_path.append(self.parse_xpath_to_gnmi_path(xpath, origin)) elif isinstance(xpaths, string_types): - gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths)] + gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths, origin)] else: raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" @@ -330,6 +296,7 @@ def subscribe_xpaths( sub_mode="SAMPLE", encoding="PROTO", sample_interval=Client._NS_IN_S * 10, + origin='openconfig' ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -393,7 +360,10 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path(xpath_subscription) + self.parse_xpath_to_gnmi_path( + xpath_subscription, + origin + ) ) subscription.mode = util.validate_proto_enum( "sub_mode", @@ -404,7 +374,10 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path(xpath_subscription["path"]) + path = self.parse_xpath_to_gnmi_path( + xpath_subscription["path"], + origin + ) arg_dict = { "path": path, "mode": sub_mode, @@ -428,18 +401,14 @@ def subscribe_xpaths( subscription_list.subscription.extend(subscriptions) return self.subscribe([subscription_list]) - def parse_xpath_to_gnmi_path(self, xpath, origin=None): + def parse_xpath_to_gnmi_path(self, xpath, origin): """Origin defaults to YANG (device) paths Otherwise specify "DME" as origin """ - if xpath.startswith("openconfig"): - raise NotImplementedError( - "OpenConfig data models not yet supported on NX-OS!" - ) if origin is None: if any(map(xpath.startswith, ["/Cisco-NX-OS-device", "/ietf-interfaces"])): origin = "device" else: origin = "DME" - origin=None + return super(NXClient, self).parse_xpath_to_gnmi_path(xpath, origin) From da34c9da9578c3da41d1b1e1154a13fd3d6ac6ef Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 16 Mar 2020 16:34:36 -0700 Subject: [PATCH 04/27] Formalized logger for nx.py --- src/cisco_gnmi/nx.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 8476e39..4dc07d9 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -31,6 +31,8 @@ from six import string_types from .client import Client, proto, util +logger = logging.getLogger(__name__) + class NXClient(Client): """NX-OS-specific wrapper for gNMI functionality. @@ -167,14 +169,14 @@ def delete_xpaths(self, xpaths, prefix=None): def check_configs(self, configs): if isinstance(configs, string_types): - logging.debug("Handling as JSON string.") + logger.debug("Handling as JSON string.") try: configs = json.loads(configs) except: raise Exception("{0}\n is invalid JSON!".format(configs)) configs = [configs] elif isinstance(configs, dict): - logging.debug("Handling already serialized JSON object.") + logger.debug("Handling already serialized JSON object.") configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( @@ -206,7 +208,6 @@ def create_updates(self, configs, origin): ) update.val.json_val = payload updates.append(update) - logging.info('GNMI set:\n\n{0}'.format(str(update))) return updates else: for config in configs: @@ -245,6 +246,8 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, raise Exception("Must supply at least one set of configurations to method!") updates = self.create_updates(update_json_configs, origin=origin) + for update in updates: + logger.info('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) replaces = self.create_updates(replace_json_configs, origin=origin) return self.set(updates=updates, replaces=replaces) @@ -287,6 +290,7 @@ def get_xpaths(self, xpaths, data_type="ALL", raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) + logger.info('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -399,6 +403,9 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) + logger.info('GNMI subscribe:\n{0}\n{1}'.format( + 15 * '=', str(subscription_list)) + ) return self.subscribe([subscription_list]) def parse_xpath_to_gnmi_path(self, xpath, origin): From 5b252161d5d10cf12aa7b55c4931b8907e58d739 Mon Sep 17 00:00:00 2001 From: miott Date: Tue, 17 Mar 2020 10:21:35 -0700 Subject: [PATCH 05/27] XE get sending correct path --- src/cisco_gnmi/nx.py | 6 ++++-- src/cisco_gnmi/xe.py | 26 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 4dc07d9..d8c1488 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -98,11 +98,13 @@ def xpath_to_path_elem(self, request): if pfx in xpath and 'openconfig' in ns: origin = 'openconfig' xpath = xpath.replace(pfx + ':', '') - value = value.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') elif pfx in xpath and 'device' in ns: origin = 'device' xpath = xpath.replace(pfx + ':', '') - value = value.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') if edit_op: if edit_op in ['create', 'merge', 'replace']: xpath_lst = xpath.split('/') diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index f39a48b..b54c857 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -29,6 +29,8 @@ from six import string_types from .client import Client, proto, util +logger = logging.getLogger(__name__) + class XEClient(Client): """IOS XE-specific wrapper for gNMI functionality. @@ -93,8 +95,6 @@ def xpath_to_path_elem(self, request): module = '' if '/Cisco-IOS-' in nspace: module = nspace[nspace.rfind('/') + 1:] - elif '/cisco-nx' in nspace: # NXOS lowercases namespace - module = 'Cisco-NX-OS-device' elif '/openconfig.net' in nspace: module = 'openconfig-' module += nspace[nspace.rfind('/') + 1:] @@ -112,8 +112,11 @@ def xpath_to_path_elem(self, request): edit_op = node.get('edit-op', '') for pfx, ns in namespace_modules.items(): - xpath = xpath.replace(pfx + ':', ns + ':') - value = value.replace(pfx + ':', ns + ':') + if pfx in xpath and 'openconfig' in ns: + origin = 'openconfig' + xpath = xpath.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') if edit_op: if edit_op in ['create', 'merge', 'replace']: xpath_lst = xpath.split('/') @@ -144,7 +147,7 @@ def xpath_to_path_elem(self, request): message['delete'] = set(xpath) else: message['get'].append(xpath) - return namespace_modules, message + return namespace_modules, message, origin def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths @@ -203,14 +206,14 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru def check_configs(configs): if isinstance(configs, string_types): - logging.debug("Handling as JSON string.") + logger.debug("Handling as JSON string.") try: configs = json.loads(configs) except: raise Exception("{0}\n is invalid JSON!".format(configs)) configs = [configs] elif isinstance(configs, dict): - logging.debug("Handling already serialized JSON object.") + logger.debug("Handling already serialized JSON object.") configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( @@ -249,7 +252,7 @@ def create_updates(configs): replaces = create_updates(replace_json_configs) return self.set(updates=updates, replaces=replaces) - def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF"): + def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None): """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. Parameters @@ -278,13 +281,16 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF"): ) gnmi_path = None if isinstance(xpaths, (list, set)): - gnmi_path = map(self.parse_xpath_to_gnmi_path, set(xpaths)) + gnmi_path = [] + for xpath in set(xpaths): + gnmi_path.append(self.parse_xpath_to_gnmi_path(xpath, origin)) elif isinstance(xpaths, string_types): - gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths)] + gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths, origin)] else: raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) + logger.info('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( From a822c79f6e79477d8106732e909fbddc73fff457 Mon Sep 17 00:00:00 2001 From: miott Date: Tue, 17 Mar 2020 12:44:34 -0700 Subject: [PATCH 06/27] XE set working --- src/cisco_gnmi/nx.py | 26 +++++-- src/cisco_gnmi/xe.py | 163 +++++++++++++++++++++++++++---------------- 2 files changed, 121 insertions(+), 68 deletions(-) diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index d8c1488..7bef265 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -186,7 +186,7 @@ def check_configs(self, configs): ) return configs - def create_updates(self, configs, origin): + def create_updates(self, configs, origin, json_ietf=False): if not configs: return None configs = self.check_configs(configs) @@ -208,7 +208,10 @@ def create_updates(self, configs, origin): xpath, origin=origin ) ) - update.val.json_val = payload + if json_ietf: + update.val.json_ietf_val = payload + else: + update.val.json_val = payload updates.append(update) return updates else: @@ -217,7 +220,7 @@ def create_updates(self, configs, origin): update = proto.gnmi_pb2.Update() update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) config = config.pop(top_element) - if ietf: + if json_ietf: update.val.json_ietf_val = json.dumps(config).encode("utf-8") else: update.val.json_val = json.dumps(config).encode("utf-8") @@ -225,7 +228,7 @@ def create_updates(self, configs, origin): return updates def set_json(self, update_json_configs=None, replace_json_configs=None, - origin='device'): + origin='device', json_ietf=False): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -247,10 +250,19 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, if not any([update_json_configs, replace_json_configs]): raise Exception("Must supply at least one set of configurations to method!") - updates = self.create_updates(update_json_configs, origin=origin) - for update in updates: + updates = self.create_updates( + update_json_configs, + origin=origin, + json_ietf=json_ietf + ) + replaces = self.create_updates( + replace_json_configs, + origin=origin, + json_ietf=json_ietf + ) + for update in updates + replaces: logger.info('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) - replaces = self.create_updates(replace_json_configs, origin=origin) + return self.set(updates=updates, replaces=replaces) def get_xpaths(self, xpaths, data_type="ALL", diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index b54c857..86a761d 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -25,6 +25,7 @@ import json import logging +import os from six import string_types from .client import Client, proto, util @@ -181,7 +182,66 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): + def check_configs(self, configs): + if isinstance(configs, string_types): + logger.debug("Handling as JSON string.") + try: + configs = json.loads(configs) + except: + raise Exception("{0}\n is invalid JSON!".format(configs)) + configs = [configs] + elif isinstance(configs, dict): + logger.debug("Handling already serialized JSON object.") + configs = [configs] + elif not isinstance(configs, (list, set)): + raise Exception( + "{0} must be an iterable of configs!".format(str(configs)) + ) + return configs + + def create_updates(self, configs, origin, json_ietf=True): + if not configs: + return None + configs = self.check_configs(configs) + + xpaths = [] + updates = [] + for config in configs: + xpath = next(iter(config.keys())) + xpaths.append(xpath) + common_xpath = os.path.commonprefix(xpaths) + + if common_xpath: + update_configs = self.get_payload(configs) + for update_cfg in update_configs: + xpath, payload = update_cfg + update = proto.gnmi_pb2.Update() + update.path.CopyFrom( + self.parse_xpath_to_gnmi_path( + xpath, origin=origin + ) + ) + if json_ietf: + update.val.json_ietf_val = payload + else: + update.val.json_val = payload + updates.append(update) + return updates + else: + for config in configs: + top_element = next(iter(config.keys())) + update = proto.gnmi_pb2.Update() + update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + config = config.pop(top_element) + if json_ietf: + update.val.json_ietf_val = json.dumps(config).encode("utf-8") + else: + update.val.json_val = json.dumps(config).encode("utf-8") + updates.append(update) + return updates + + def set_json(self, update_json_configs=None, replace_json_configs=None, + origin='device', json_ietf=True): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -194,8 +254,7 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru JSON configs to apply as updates. replace_json_configs : iterable of JSON configurations, optional JSON configs to apply as replacements. - ietf : bool, optional - Use JSON_IETF vs JSON. + origin : openconfig, device, or DME Returns ------- @@ -204,52 +263,19 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru if not any([update_json_configs, replace_json_configs]): raise Exception("Must supply at least one set of configurations to method!") - def check_configs(configs): - if isinstance(configs, string_types): - logger.debug("Handling as JSON string.") - try: - configs = json.loads(configs) - except: - raise Exception("{0}\n is invalid JSON!".format(configs)) - configs = [configs] - elif isinstance(configs, dict): - logger.debug("Handling already serialized JSON object.") - configs = [configs] - elif not isinstance(configs, (list, set)): - raise Exception( - "{0} must be an iterable of configs!".format(str(configs)) - ) - return configs - - def create_updates(configs): - if not configs: - return None - configs = check_configs(configs) - updates = [] - for config in configs: - if not isinstance(config, dict): - raise Exception("config must be a JSON object!") - if len(config.keys()) > 1: - raise Exception("config should only target one YANG module!") - top_element = next(iter(config.keys())) - # start mike - # path_obj = self.parse_xpath_to_gnmi_path(top_element) - # config = config.pop(top_element) - # value_obj = proto.gnmi_pb2.TypedValue(json_ietf_val=json.dumps(config).encode("utf-8")) - # update = proto.gnmi_pb2.Update(path=path_obj, val=value_obj) - # end mike - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) - config = config.pop(top_element) - if ietf: - update.val.json_ietf_val = json.dumps(config).encode("utf-8") - else: - update.val.json_val = json.dumps(config).encode("utf-8") - updates.append(update) - return updates + updates = self.create_updates( + update_json_configs, + origin=origin, + json_ietf=json_ietf + ) + replaces = self.create_updates( + replace_json_configs, + origin=origin, + json_ietf=json_ietf + ) + for update in updates + replaces: + logger.info('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) - updates = create_updates(update_json_configs) - replaces = create_updates(replace_json_configs) return self.set(updates=updates, replaces=replaces) def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None): @@ -296,9 +322,11 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None) def subscribe_xpaths( self, xpath_subscriptions, - encoding="JSON_IETF", + request_mode="STREAM", + sub_mode="SAMPLE", + encoding="PROTO", sample_interval=Client._NS_IN_S * 10, - heartbeat_interval=None, + origin='openconfig' ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -315,26 +343,30 @@ def subscribe_xpaths( to SubscriptionRequest. Strings are parsed as XPaths and defaulted with the default arguments, dictionaries are treated as dicts of args to pass to the Subscribe init, and Subscription is treated as simply a pre-made Subscription. + request_mode : proto.gnmi_pb2.SubscriptionList.Mode, optional + Indicates whether STREAM to stream from target, + ONCE to stream once (like a get), + POLL to respond to POLL. + [STREAM, ONCE, POLL] + sub_mode : proto.gnmi_pb2.SubscriptionMode, optional + The default SubscriptionMode on a per Subscription basis in the SubscriptionList. + ON_CHANGE only streams updates when changes occur. + SAMPLE will stream the subscription at a regular cadence/interval. + [ON_CHANGE, SAMPLE] encoding : proto.gnmi_pb2.Encoding, optional A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data - [JSON, JSON_IETF] + [JSON, PROTO] sample_interval : int, optional Default nanoseconds for sample to occur. Defaults to 10 seconds. - heartbeat_interval : int, optional - Specifies the maximum allowable silent period in nanoseconds when - suppress_redundant is in use. The target should send a value at least once - in the period specified. Returns ------- subscribe() """ - supported_request_modes = ["STREAM"] - request_mode = "STREAM" - supported_sub_modes = ["SAMPLE"] - sub_mode = "SAMPLE" + supported_request_modes = ["STREAM", "ONCE", "POLL"] supported_encodings = ["JSON", "JSON_IETF"] + supported_sub_modes = ["ON_CHANGE", "SAMPLE"] subscription_list = proto.gnmi_pb2.SubscriptionList() subscription_list.mode = util.validate_proto_enum( "mode", @@ -358,7 +390,10 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path(xpath_subscription) + self.parse_xpath_to_gnmi_path( + xpath_subscription, + origin + ) ) subscription.mode = util.validate_proto_enum( "sub_mode", @@ -369,7 +404,10 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path(xpath_subscription["path"]) + path = self.parse_xpath_to_gnmi_path( + xpath_subscription["path"], + origin + ) arg_dict = { "path": path, "mode": sub_mode, @@ -391,6 +429,9 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) + logger.info('GNMI subscribe:\n{0}\n{1}'.format( + 15 * '=', str(subscription_list)) + ) return self.subscribe([subscription_list]) def parse_xpath_to_gnmi_path(self, xpath, origin=None): From 1576b0ca1ea2bff7ba048ab5f6fbf39bbc6abafa Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 20 Mar 2020 14:51:11 -0700 Subject: [PATCH 07/27] Fix logger --- src/cisco_gnmi/xe.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 86a761d..33d59f7 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -92,6 +92,7 @@ def xpath_to_path_elem(self, request): return paths else: namespace_modules = {} + origin = None for prefix, nspace in request.get('namespace', {}).items(): module = '' if '/Cisco-IOS-' in nspace: @@ -106,7 +107,7 @@ def xpath_to_path_elem(self, request): namespace_modules[prefix] = module for node in request.get('nodes', []): if 'xpath' not in node: - log.error('Xpath is not in message') + logger.error('Xpath is not in message') else: xpath = node['xpath'] value = node.get('value', '') From 8ef01a1db1ffeacb7aa8fdeb5894f4fb7d04b281 Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 23 Mar 2020 13:43:04 -0700 Subject: [PATCH 08/27] Changed return of empty updates --- src/cisco_gnmi/nx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 7bef265..20ee938 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -188,7 +188,7 @@ def check_configs(self, configs): def create_updates(self, configs, origin, json_ietf=False): if not configs: - return None + return [] configs = self.check_configs(configs) xpaths = [] From 197d232a10426e066176dab441e03b1755a7827e Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 23 Mar 2020 14:48:28 -0700 Subject: [PATCH 09/27] Add docstrings --- src/cisco_gnmi/client.py | 31 ++++++++++++++++++++++++++++++ src/cisco_gnmi/nx.py | 41 ++++++++++++++++++++++++++++++++++++++++ src/cisco_gnmi/xe.py | 40 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index ddae60c..982ae81 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -342,6 +342,19 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): return path def combine_configs(self, payload, last_xpath, xpath, config): + """Walking from end to finish 2 xpaths merge so combine them + + |---last xpath config + ----| + |---this xpath config + + Parameters + ---------- + payload: dict of partial payload + last_xpath: last xpath that was processed + xpath: colliding xpath + config: dict of values associated to colliding xpath + """ last_set = set(last_xpath.split('/')) curr_diff = set(xpath.split('/')) - last_set if len(curr_diff) > 1: @@ -368,6 +381,18 @@ def combine_configs(self, payload, last_xpath, xpath, config): return payload def xpath_to_json(self, configs, last_xpath='', payload={}): + """Try to combine Xpaths/values into a common payload (recursive). + + Parameters + ---------- + configs: tuple of xpath/value dict + last_xpath: str of last xpath that was recusivly processed. + payload: dict being recursively built for JSON transformation. + + Returns + ------- + dict of combined xpath/value dict. + """ for i, cfg in enumerate(configs, 1): xpath, config, is_key = cfg if last_xpath and xpath not in last_xpath: @@ -408,6 +433,12 @@ def xpath_to_json(self, configs, last_xpath='', payload={}): RE_FIND_KEYS = re.compile(r'\[.*?\]') def get_payload(self, configs): + """Common Xpaths were detected so try to consolidate them. + + Parameter + --------- + configs: tuple of xpath/value dicts + """ # Number of updates are limited so try to consolidate into lists. xpaths_cfg = [] first_key = set() diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 20ee938..c12efe6 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -58,6 +58,27 @@ class NXClient(Client): """ def xpath_to_path_elem(self, request): + """Helper function for Nexus clients. + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: DME, device, or openconfig + """ + paths = [] message = { 'update': [], @@ -187,6 +208,26 @@ def check_configs(self, configs): return configs def create_updates(self, configs, origin, json_ietf=False): + """Check configs, and construct "Update" messages. + + Parameters + ---------- + configs: dict of : + origin: str [DME, device, openconfig] + json_ietf: bool encoding type for Update val (default False) + + Returns + ------- + List of Update messages with val populated. + + If a set of configs contain a common Xpath, the Update must contain + a consolidation of xpath/values for 2 reasons: + + 1. Devices may have a restriction on how many Update messages it will + accept at once. + 2. Some xpath/values are required to be set in same Update because of + dependencies like leafrefs, mandatory settings, and if/when/musts. + """ if not configs: return [] configs = self.check_configs(configs) diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 33d59f7..dab8ff3 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -80,6 +80,26 @@ class XEClient(Client): """ def xpath_to_path_elem(self, request): + """Helper function for Cisco XE clients. + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: openconfig if detected (XE uses rfc7951 by default) + """ paths = [] message = { 'update': [], @@ -201,6 +221,26 @@ def check_configs(self, configs): return configs def create_updates(self, configs, origin, json_ietf=True): + """Check configs, and construct "Update" messages. + + Parameters + ---------- + configs: dict of : + origin: None or 'openconfig' + json_ietf: bool encoding type for Update val (default True) + + Returns + ------- + List of Update messages with val populated. + + If a set of configs contain a common Xpath, the Update must contain + a consolidation of xpath/values for 2 reasons: + + 1. Devices may have a restriction on how many Update messages it will + accept at once. + 2. Some xpath/values are required to be set in same Update because of + dependencies like leafrefs, mandatory settings, and if/when/musts. + """ if not configs: return None configs = self.check_configs(configs) From 53aed630275778522fd2cc8f5374a90b2acffead Mon Sep 17 00:00:00 2001 From: miott Date: Thu, 9 Apr 2020 16:26:55 -0700 Subject: [PATCH 10/27] XE testbed working with review comment changes. --- src/cisco_gnmi/client.py | 309 ++++++++++++++++++++++++++++++++++++++- src/cisco_gnmi/nx.py | 217 +++++---------------------- src/cisco_gnmi/util.py | 1 - src/cisco_gnmi/xe.py | 135 +++++------------ 4 files changed, 378 insertions(+), 284 deletions(-) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 982ae81..dbe2deb 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -31,8 +31,8 @@ import os from six import string_types -from . import proto -from . import util +from cisco_gnmi import proto +from cisco_gnmi import util class Client(object): @@ -257,6 +257,84 @@ def validate_request(request): ) return response_stream + def check_configs(self, configs): + if isinstance(configs, string_types): + logger.debug("Handling as JSON string.") + try: + configs = json.loads(configs) + except: + raise Exception("{0}\n is invalid JSON!".format(configs)) + configs = [configs] + elif isinstance(configs, dict): + logger.debug("Handling already serialized JSON object.") + configs = [configs] + elif not isinstance(configs, (list, set)): + raise Exception( + "{0} must be an iterable of configs!".format(str(configs)) + ) + return configs + + def create_updates(self, configs, origin, json_ietf=False): + """Check configs, and construct "Update" messages. + + Parameters + ---------- + configs: dict of : + origin: str [DME, device, openconfig] + json_ietf: bool encoding type for Update val (default False) + + Returns + ------- + List of Update messages with val populated. + + If a set of configs contain a common Xpath, the Update must contain + a consolidation of xpath/values for 2 reasons: + + 1. Devices may have a restriction on how many Update messages it will + accept at once. + 2. Some xpath/values are required to be set in same Update because of + dependencies like leafrefs, mandatory settings, and if/when/musts. + """ + if not configs: + return [] + configs = self.check_configs(configs) + + xpaths = [] + updates = [] + for config in configs: + xpath = next(iter(config.keys())) + xpaths.append(xpath) + common_xpath = os.path.commonprefix(xpaths) + + if common_xpath: + update_configs = self.get_payload(configs) + for update_cfg in update_configs: + xpath, payload = update_cfg + update = proto.gnmi_pb2.Update() + update.path.CopyFrom( + self.parse_xpath_to_gnmi_path( + xpath, origin=origin + ) + ) + if json_ietf: + update.val.json_ietf_val = payload + else: + update.val.json_val = payload + updates.append(update) + return updates + else: + for config in configs: + top_element = next(iter(config.keys())) + update = proto.gnmi_pb2.Update() + update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + config = config.pop(top_element) + if json_ietf: + update.val.json_ietf_val = json.dumps(config).encode("utf-8") + else: + update.val.json_val = json.dumps(config).encode("utf-8") + updates.append(update) + return updates + def parse_xpath_to_gnmi_path(self, xpath, origin=None): """Parses an XPath to proto.gnmi_pb2.Path. This function should be overridden by any child classes for origin logic. @@ -540,3 +618,230 @@ def get_payload(self, configs): ) ) return updates + + def xml_path_to_path_elem(self, request): + """Convert XML Path Language 1.0 Xpath to gNMI Path/PathElement. + + Modeled after YANG/NETCONF Xpaths. + + References: + * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths + * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev + * https://tools.ietf.org/html/rfc6020#section-6.4 + * https://tools.ietf.org/html/rfc6020#section-9.13 + * https://tools.ietf.org/html/rfc6241 + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: DME, device, or openconfig + """ + + paths = [] + message = { + 'update': [], + 'replace': [], + 'delete': [], + 'get': [], + } + if 'nodes' not in request: + # TODO: raw rpc? + return paths + else: + namespace_modules = {} + origin = 'DME' + for prefix, nspace in request.get('namespace', {}).items(): + if '/Cisco-IOS-' in nspace: + module = nspace[nspace.rfind('/') + 1:] + elif '/cisco-nx' in nspace: # NXOS lowercases namespace + module = 'Cisco-NX-OS-device' + elif '/openconfig.net' in nspace: + module = 'openconfig-' + module += nspace[nspace.rfind('/') + 1:] + elif 'urn:ietf:params:xml:ns:yang:' in nspace: + module = nspace.replace( + 'urn:ietf:params:xml:ns:yang:', '') + if module: + namespace_modules[prefix] = module + + for node in request.get('nodes', []): + if 'xpath' not in node: + log.error('Xpath is not in message') + else: + xpath = node['xpath'] + value = node.get('value', '') + edit_op = node.get('edit-op', '') + + for pfx, ns in namespace_modules.items(): + # NXOS does not support prefixes yet so clear them out + if pfx in xpath and 'openconfig' in ns: + origin = 'openconfig' + xpath = xpath.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') + elif pfx in xpath and 'device' in ns: + origin = 'device' + xpath = xpath.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') + if edit_op: + if edit_op in ['create', 'merge', 'replace']: + xpath_lst = xpath.split('/') + name = xpath_lst.pop() + xpath = '/'.join(xpath_lst) + if edit_op == 'replace': + if not message['replace']: + message['replace'] = [{ + xpath: {name: value} + }] + else: + message['replace'].append( + {xpath: {name: value}} + ) + else: + if not message['update']: + message['update'] = [{ + xpath: {name: value} + }] + else: + message['update'].append( + {xpath: {name: value}} + ) + elif edit_op in ['delete', 'remove']: + if message['delete']: + message['delete'].add(xpath) + else: + message['delete'] = set(xpath) + else: + message['get'].append(xpath) + return namespace_modules, message, origin + + +if __name__ == '__main__': + from pprint import pprint as pp + import grpc + from cisco_gnmi import Client + from cisco_gnmi.auth import CiscoAuthPlugin + channel = grpc.secure_channel( + '127.0.0.1:9339', + grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials( + CiscoAuthPlugin( + 'admin', + 'its_a_secret' + ) + ) + ) + ) + client = Client(channel) + request = { + 'namespace': { + 'oc-acl': 'http://openconfig.net/yang/acl' + }, + 'nodes': [ + { + 'value': 'testacl', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', + 'edit-op': 'merge' + }, + { + 'value': 'ACL_IPV4', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', + 'edit-op': 'merge' + }, + { + 'value': '10', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', + 'edit-op': 'merge' + }, + { + 'value': '20.20.20.1/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', + 'edit-op': 'merge' + }, + { + 'value': 'IP_TCP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', + 'edit-op': 'merge' + }, + { + 'value': '10.10.10.10/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', + 'edit-op': 'merge' + }, + { + 'value': 'DROP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', + 'edit-op': 'merge' + } + ] + } + modules, message, origin = client.xpath_to_path_elem(request) + pp(modules) + pp(message) + pp(origin) + """ + # Expected output + ================= + {'oc-acl': 'openconfig-acl'} + {'delete': [], + 'get': [], + 'replace': [], + 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, + {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}]} + 'openconfig' + """ + # Feed converted XML Path Language 1.0 Xpaths to create updates + updates = client.create_updates(message['update'], origin) + pp(updates) + """ + # Expected output + ================= + [path { + origin: "openconfig" + elem { + name: "acl" + } + elem { + name: "acl-sets" + } + elem { + name: "acl-set" + key { + key: "name" + value: "testacl" + } + key { + key: "type" + value: "ACL_IPV4" + } + } + elem { + name: "acl-entries" + } + } + val { + json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" + } + ] + # update is now ready to be sent through gNMI SetRequest + """ diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index c12efe6..f18610b 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -56,108 +56,6 @@ class NXClient(Client): >>> capabilities = client.capabilities() >>> print(capabilities) """ - - def xpath_to_path_elem(self, request): - """Helper function for Nexus clients. - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: DME, device, or openconfig - """ - - paths = [] - message = { - 'update': [], - 'replace': [], - 'delete': [], - 'get': [], - } - if 'nodes' not in request: - # TODO: raw rpc? - return paths - else: - namespace_modules = {} - origin = 'DME' - for prefix, nspace in request.get('namespace', {}).items(): - if '/Cisco-IOS-' in nspace: - module = nspace[nspace.rfind('/') + 1:] - elif '/cisco-nx' in nspace: # NXOS lowercases namespace - module = 'Cisco-NX-OS-device' - elif '/openconfig.net' in nspace: - module = 'openconfig-' - module += nspace[nspace.rfind('/') + 1:] - elif 'urn:ietf:params:xml:ns:yang:' in nspace: - module = nspace.replace( - 'urn:ietf:params:xml:ns:yang:', '') - if module: - namespace_modules[prefix] = module - - for node in request.get('nodes', []): - if 'xpath' not in node: - log.error('Xpath is not in message') - else: - xpath = node['xpath'] - value = node.get('value', '') - edit_op = node.get('edit-op', '') - - for pfx, ns in namespace_modules.items(): - # NXOS does not support prefixes yet so clear them out - if pfx in xpath and 'openconfig' in ns: - origin = 'openconfig' - xpath = xpath.replace(pfx + ':', '') - if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - elif pfx in xpath and 'device' in ns: - origin = 'device' - xpath = xpath.replace(pfx + ':', '') - if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - if edit_op: - if edit_op in ['create', 'merge', 'replace']: - xpath_lst = xpath.split('/') - name = xpath_lst.pop() - xpath = '/'.join(xpath_lst) - if edit_op == 'replace': - if not message['replace']: - message['replace'] = [{ - xpath: {name: value} - }] - else: - message['replace'].append( - {xpath: {name: value}} - ) - else: - if not message['update']: - message['update'] = [{ - xpath: {name: value} - }] - else: - message['update'].append( - {xpath: {name: value}} - ) - elif edit_op in ['delete', 'remove']: - if message['delete']: - message['delete'].add(xpath) - else: - message['delete'] = set(xpath) - else: - message['get'].append(xpath) - return namespace_modules, message, origin - def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. @@ -190,84 +88,6 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def check_configs(self, configs): - if isinstance(configs, string_types): - logger.debug("Handling as JSON string.") - try: - configs = json.loads(configs) - except: - raise Exception("{0}\n is invalid JSON!".format(configs)) - configs = [configs] - elif isinstance(configs, dict): - logger.debug("Handling already serialized JSON object.") - configs = [configs] - elif not isinstance(configs, (list, set)): - raise Exception( - "{0} must be an iterable of configs!".format(str(configs)) - ) - return configs - - def create_updates(self, configs, origin, json_ietf=False): - """Check configs, and construct "Update" messages. - - Parameters - ---------- - configs: dict of : - origin: str [DME, device, openconfig] - json_ietf: bool encoding type for Update val (default False) - - Returns - ------- - List of Update messages with val populated. - - If a set of configs contain a common Xpath, the Update must contain - a consolidation of xpath/values for 2 reasons: - - 1. Devices may have a restriction on how many Update messages it will - accept at once. - 2. Some xpath/values are required to be set in same Update because of - dependencies like leafrefs, mandatory settings, and if/when/musts. - """ - if not configs: - return [] - configs = self.check_configs(configs) - - xpaths = [] - updates = [] - for config in configs: - xpath = next(iter(config.keys())) - xpaths.append(xpath) - common_xpath = os.path.commonprefix(xpaths) - - if common_xpath: - update_configs = self.get_payload(configs) - for update_cfg in update_configs: - xpath, payload = update_cfg - update = proto.gnmi_pb2.Update() - update.path.CopyFrom( - self.parse_xpath_to_gnmi_path( - xpath, origin=origin - ) - ) - if json_ietf: - update.val.json_ietf_val = payload - else: - update.val.json_val = payload - updates.append(update) - return updates - else: - for config in configs: - top_element = next(iter(config.keys())) - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) - config = config.pop(top_element) - if json_ietf: - update.val.json_ietf_val = json.dumps(config).encode("utf-8") - else: - update.val.json_val = json.dumps(config).encode("utf-8") - updates.append(update) - return updates - def set_json(self, update_json_configs=None, replace_json_configs=None, origin='device', json_ietf=False): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. @@ -302,7 +122,7 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, json_ietf=json_ietf ) for update in updates + replaces: - logger.info('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) + logger.debug('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) return self.set(updates=updates, replaces=replaces) @@ -345,7 +165,7 @@ def get_xpaths(self, xpaths, data_type="ALL", raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) - logger.info('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) + logger.debug('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -458,7 +278,7 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) - logger.info('GNMI subscribe:\n{0}\n{1}'.format( + logger.debug('GNMI subscribe:\n{0}\n{1}'.format( 15 * '=', str(subscription_list)) ) return self.subscribe([subscription_list]) @@ -474,3 +294,34 @@ def parse_xpath_to_gnmi_path(self, xpath, origin): origin = "DME" return super(NXClient, self).parse_xpath_to_gnmi_path(xpath, origin) + + def xpath_to_path_elem(self, request): + """Convert XML Path Language 1.0 formed xpath to gNMI PathElement. + + Modeled after NETCONF Xpaths RFC 6020 (See client.py for use example). + + References: + * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths + * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev + * https://tools.ietf.org/html/rfc6020#section-6.4 + * https://tools.ietf.org/html/rfc6020#section-9.13 + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: DME, device, or openconfig + """ + return super(NXClient, self).xml_path_to_path_elem(request) diff --git a/src/cisco_gnmi/util.py b/src/cisco_gnmi/util.py index bca78a7..f65f104 100644 --- a/src/cisco_gnmi/util.py +++ b/src/cisco_gnmi/util.py @@ -110,7 +110,6 @@ def get_cn_from_cert(cert_pem): Defaults to first found if multiple CNs identified. """ cert_cn = None - import pdb; pdb.set_trace() cert_parsed = x509.load_pem_x509_certificate(cert_pem, default_backend()) cert_cns = cert_parsed.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME) if len(cert_cns) > 0: diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index dab8ff3..f7c2386 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -31,6 +31,7 @@ from .client import Client, proto, util logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) class XEClient(Client): @@ -78,99 +79,6 @@ class XEClient(Client): ... >>> delete_response = client.delete_xpaths('/Cisco-IOS-XE-native:native/hostname') """ - - def xpath_to_path_elem(self, request): - """Helper function for Cisco XE clients. - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: openconfig if detected (XE uses rfc7951 by default) - """ - paths = [] - message = { - 'update': [], - 'replace': [], - 'delete': [], - 'get': [], - } - if 'nodes' not in request: - # TODO: raw rpc? - return paths - else: - namespace_modules = {} - origin = None - for prefix, nspace in request.get('namespace', {}).items(): - module = '' - if '/Cisco-IOS-' in nspace: - module = nspace[nspace.rfind('/') + 1:] - elif '/openconfig.net' in nspace: - module = 'openconfig-' - module += nspace[nspace.rfind('/') + 1:] - elif 'urn:ietf:params:xml:ns:yang:' in nspace: - module = nspace.replace( - 'urn:ietf:params:xml:ns:yang:', '') - if module: - namespace_modules[prefix] = module - for node in request.get('nodes', []): - if 'xpath' not in node: - logger.error('Xpath is not in message') - else: - xpath = node['xpath'] - value = node.get('value', '') - edit_op = node.get('edit-op', '') - - for pfx, ns in namespace_modules.items(): - if pfx in xpath and 'openconfig' in ns: - origin = 'openconfig' - xpath = xpath.replace(pfx + ':', '') - if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - if edit_op: - if edit_op in ['create', 'merge', 'replace']: - xpath_lst = xpath.split('/') - name = xpath_lst.pop() - xpath = '/'.join(xpath_lst) - if edit_op == 'replace': - if not message['replace']: - message['replace'] = [{ - xpath: {name: value} - }] - else: - message['replace'].append( - {xpath: {name: value}} - ) - else: - if not message['update']: - message['update'] = [{ - xpath: {name: value} - }] - else: - message['update'].append( - {xpath: {name: value}} - ) - elif edit_op in ['delete', 'remove']: - if message['delete']: - message['delete'].add(xpath) - else: - message['delete'] = set(xpath) - else: - message['get'].append(xpath) - return namespace_modules, message, origin - def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. @@ -315,7 +223,7 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, json_ietf=json_ietf ) for update in updates + replaces: - logger.info('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) + logger.debug('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) return self.set(updates=updates, replaces=replaces) @@ -357,7 +265,7 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None) raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) - logger.info('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) + logger.debug('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -365,7 +273,7 @@ def subscribe_xpaths( xpath_subscriptions, request_mode="STREAM", sub_mode="SAMPLE", - encoding="PROTO", + encoding="JSON_IETF", sample_interval=Client._NS_IN_S * 10, origin='openconfig' ): @@ -396,7 +304,7 @@ def subscribe_xpaths( [ON_CHANGE, SAMPLE] encoding : proto.gnmi_pb2.Encoding, optional A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data - [JSON, PROTO] + [JSON, JSON_IETF] sample_interval : int, optional Default nanoseconds for sample to occur. Defaults to 10 seconds. @@ -470,7 +378,7 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) - logger.info('GNMI subscribe:\n{0}\n{1}'.format( + logger.debug('GNMI subscribe:\n{0}\n{1}'.format( 15 * '=', str(subscription_list)) ) return self.subscribe([subscription_list]) @@ -487,3 +395,34 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): else: origin = "rfc7951" return super(XEClient, self).parse_xpath_to_gnmi_path(xpath, origin) + + def xpath_to_path_elem(self, request): + """Convert XML Path Language 1.0 formed xpath to gNMI PathElement. + + Modeled after NETCONF Xpaths RFC 6020. + + References: + * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths + * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev + * https://tools.ietf.org/html/rfc6020#section-6.4 + * https://tools.ietf.org/html/rfc6020#section-9.13 + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: DME, device, or openconfig + """ + return super(XEClient, self).xml_path_to_path_elem(request) From 451c0138a18cc59b395a2057774a6c25115ceab8 Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 10 Apr 2020 10:17:51 -0700 Subject: [PATCH 11/27] Refactored combine function and added comments. --- src/cisco_gnmi/client.py | 61 +++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index dbe2deb..ea3d988 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -419,13 +419,15 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): path.elem.extend(path_elems) return path - def combine_configs(self, payload, last_xpath, xpath, config): + def combine_configs(self, payload, last_xpath, cfg): """Walking from end to finish 2 xpaths merge so combine them - - |---last xpath config - ----| - |---this xpath config - + |--config + |---last xpath config--| + ----| |--config + | + | pick these up --> |--config + |---this xpath config--| + |--config Parameters ---------- payload: dict of partial payload @@ -433,29 +435,19 @@ def combine_configs(self, payload, last_xpath, xpath, config): xpath: colliding xpath config: dict of values associated to colliding xpath """ - last_set = set(last_xpath.split('/')) - curr_diff = set(xpath.split('/')) - last_set - if len(curr_diff) > 1: - print('combine_configs() error1') - return payload - index = curr_diff.pop() - curr_xpath = xpath[xpath.find(index):] - curr_xpath = curr_xpath.split('/') - curr_xpath.reverse() - for seg in curr_xpath: - config = {seg: config} - - last_diff = last_set - set(xpath.split('/')) - if len(last_diff) > 1: - print('combine_configs() error2') - return payload - last_xpath = last_xpath[last_xpath.find(last_diff.pop()):] - last_xpath = last_xpath.split('/') - last_xpath.reverse() - for seg in last_xpath: - if seg not in payload: - payload = {seg: payload} - payload.update(config) + xpath, config, is_key = cfg + lp = last_xpath.split('/') + xp = xpath.split('/') + base = [] + top = '' + for i, seg in enumerate(zip(lp, xp)): + if seg[0] != seg[1]: + top = seg[1] + break + base = '/' + '/'.join(xp[i:]) + cfg = (base, config, False) + extended_payload = {top: self.xpath_to_json([cfg])} + payload.update(extended_payload) return payload def xpath_to_json(self, configs, last_xpath='', payload={}): @@ -474,11 +466,11 @@ def xpath_to_json(self, configs, last_xpath='', payload={}): for i, cfg in enumerate(configs, 1): xpath, config, is_key = cfg if last_xpath and xpath not in last_xpath: - # Branched config here - # |---last xpath config - # --| + # Branched config here |---config + # |---last xpath config--| + # --| |---config # |---this xpath config - payload = self.combine_configs(payload, last_xpath, xpath, config) + payload = self.combine_configs(payload, last_xpath, cfg) return self.xpath_to_json(configs[i:], xpath, payload) xpath_segs = xpath.split('/') xpath_segs.reverse() @@ -733,7 +725,6 @@ def xml_path_to_path_elem(self, request): if __name__ == '__main__': from pprint import pprint as pp import grpc - from cisco_gnmi import Client from cisco_gnmi.auth import CiscoAuthPlugin channel = grpc.secure_channel( '127.0.0.1:9339', @@ -790,7 +781,7 @@ def xml_path_to_path_elem(self, request): } ] } - modules, message, origin = client.xpath_to_path_elem(request) + modules, message, origin = client.xml_path_to_path_elem(request) pp(modules) pp(message) pp(origin) From 803f556e65285091965b92a57daf98464671a7f1 Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 10 Apr 2020 10:36:26 -0700 Subject: [PATCH 12/27] Removed functions from xe subclass --- src/cisco_gnmi/xe.py | 78 -------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index f7c2386..59af5de 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -111,84 +111,6 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def check_configs(self, configs): - if isinstance(configs, string_types): - logger.debug("Handling as JSON string.") - try: - configs = json.loads(configs) - except: - raise Exception("{0}\n is invalid JSON!".format(configs)) - configs = [configs] - elif isinstance(configs, dict): - logger.debug("Handling already serialized JSON object.") - configs = [configs] - elif not isinstance(configs, (list, set)): - raise Exception( - "{0} must be an iterable of configs!".format(str(configs)) - ) - return configs - - def create_updates(self, configs, origin, json_ietf=True): - """Check configs, and construct "Update" messages. - - Parameters - ---------- - configs: dict of : - origin: None or 'openconfig' - json_ietf: bool encoding type for Update val (default True) - - Returns - ------- - List of Update messages with val populated. - - If a set of configs contain a common Xpath, the Update must contain - a consolidation of xpath/values for 2 reasons: - - 1. Devices may have a restriction on how many Update messages it will - accept at once. - 2. Some xpath/values are required to be set in same Update because of - dependencies like leafrefs, mandatory settings, and if/when/musts. - """ - if not configs: - return None - configs = self.check_configs(configs) - - xpaths = [] - updates = [] - for config in configs: - xpath = next(iter(config.keys())) - xpaths.append(xpath) - common_xpath = os.path.commonprefix(xpaths) - - if common_xpath: - update_configs = self.get_payload(configs) - for update_cfg in update_configs: - xpath, payload = update_cfg - update = proto.gnmi_pb2.Update() - update.path.CopyFrom( - self.parse_xpath_to_gnmi_path( - xpath, origin=origin - ) - ) - if json_ietf: - update.val.json_ietf_val = payload - else: - update.val.json_val = payload - updates.append(update) - return updates - else: - for config in configs: - top_element = next(iter(config.keys())) - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) - config = config.pop(top_element) - if json_ietf: - update.val.json_ietf_val = json.dumps(config).encode("utf-8") - else: - update.val.json_val = json.dumps(config).encode("utf-8") - updates.append(update) - return updates - def set_json(self, update_json_configs=None, replace_json_configs=None, origin='device', json_ietf=True): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. From 34b39af27cb053a54292d96e8c3b42911ef446f0 Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 10 Apr 2020 12:16:32 -0700 Subject: [PATCH 13/27] Resolved confict and addressed review comments. --- Makefile | 18 ------------------ src/cisco_gnmi/client.py | 4 +--- src/cisco_gnmi/xe.py | 2 +- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 47611af..1829293 100644 --- a/Makefile +++ b/Makefile @@ -85,21 +85,3 @@ help: } \ }' \ $(MAKEFILE_LIST) - -## Setup links in virtual env for development -develop: - @echo "--------------------------------------------------------------------" - @echo "Setting up development environment" - @python setup.py develop -q - @echo "" - @echo "Done." - @echo "" - -## Remove development links in virtual env -undevelop: - @echo "--------------------------------------------------------------------" - @echo "Removing development environment" - @python setup.py develop -q --uninstall - @echo "" - @echo "Done." - @echo "" \ No newline at end of file diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index ea3d988..5b0961f 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -156,9 +156,7 @@ def get( "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding ) request = proto.gnmi_pb2.GetRequest() - try: - iter(paths) - except TypeError: + if not isinstance(paths, (list, set, map)): raise Exception("paths must be an iterable containing Path(s)!") request.path.extend(paths) request.type = data_type diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 59af5de..c7775c4 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -312,7 +312,7 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): """ if origin is None: # naive but effective - if "openconfig" in xpath: + if ":" in xpath: origin = "openconfig" else: origin = "rfc7951" From d36caa672d620f50edf54ac4d46661b882309d0b Mon Sep 17 00:00:00 2001 From: miott Date: Fri, 10 Apr 2020 15:35:40 -0700 Subject: [PATCH 14/27] Moved xpath parsing to xpath_util.py --- src/cisco_gnmi/__init__.py | 1 + src/cisco_gnmi/client.py | 509 +--------------------------------- src/cisco_gnmi/nx.py | 57 +--- src/cisco_gnmi/xe.py | 58 +--- src/cisco_gnmi/xpath_util.py | 516 +++++++++++++++++++++++++++++++++++ 5 files changed, 537 insertions(+), 604 deletions(-) create mode 100644 src/cisco_gnmi/xpath_util.py diff --git a/src/cisco_gnmi/__init__.py b/src/cisco_gnmi/__init__.py index 0005330..a0c51f8 100644 --- a/src/cisco_gnmi/__init__.py +++ b/src/cisco_gnmi/__init__.py @@ -29,5 +29,6 @@ from .nx import NXClient from .xe import XEClient from .builder import ClientBuilder +from . import xpath_util __version__ = "1.0.4" diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 5b0961f..796b3f1 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -33,6 +33,7 @@ from cisco_gnmi import proto from cisco_gnmi import util +from cisco_gnmi.xpath_util import get_payload, parse_xpath_to_gnmi_path class Client(object): @@ -305,12 +306,12 @@ def create_updates(self, configs, origin, json_ietf=False): common_xpath = os.path.commonprefix(xpaths) if common_xpath: - update_configs = self.get_payload(configs) + update_configs = get_payload(configs) for update_cfg in update_configs: xpath, payload = update_cfg update = proto.gnmi_pb2.Update() update.path.CopyFrom( - self.parse_xpath_to_gnmi_path( + parse_xpath_to_gnmi_path( xpath, origin=origin ) ) @@ -324,7 +325,7 @@ def create_updates(self, configs, origin, json_ietf=False): for config in configs: top_element = next(iter(config.keys())) update = proto.gnmi_pb2.Update() - update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + update.path.CopyFrom(parse_xpath_to_gnmi_path(top_element)) config = config.pop(top_element) if json_ietf: update.val.json_ietf_val = json.dumps(config).encode("utf-8") @@ -332,505 +333,3 @@ def create_updates(self, configs, origin, json_ietf=False): update.val.json_val = json.dumps(config).encode("utf-8") updates.append(update) return updates - - def parse_xpath_to_gnmi_path(self, xpath, origin=None): - """Parses an XPath to proto.gnmi_pb2.Path. - This function should be overridden by any child classes for origin logic. - - Effectively wraps the std XML XPath tokenizer and traverses - the identified groups. Parsing robustness needs to be validated. - Probably best to formalize as a state machine sometime. - TODO: Formalize tokenizer traversal via state machine. - """ - if not isinstance(xpath, string_types): - raise Exception("xpath must be a string!") - path = proto.gnmi_pb2.Path() - if origin: - if not isinstance(origin, string_types): - raise Exception("origin must be a string!") - path.origin = origin - curr_elem = proto.gnmi_pb2.PathElem() - in_filter = False - just_filtered = False - curr_key = None - # TODO: Lazy - xpath = xpath.strip("/") - xpath_elements = xpath_tokenizer_re.findall(xpath) - path_elems = [] - for index, element in enumerate(xpath_elements): - # stripped initial /, so this indicates a completed element - if element[0] == "/": - if not curr_elem.name: - raise Exception( - "Current PathElem has no name yet is trying to be pushed to path! Invalid XPath?" - ) - path_elems.append(curr_elem) - curr_elem = proto.gnmi_pb2.PathElem() - continue - # We are entering a filter - elif element[0] == "[": - in_filter = True - continue - # We are exiting a filter - elif element[0] == "]": - in_filter = False - continue - # If we're not in a filter then we're a PathElem name - elif not in_filter: - curr_elem.name = element[1] - # Skip blank spaces - elif not any([element[0], element[1]]): - continue - # If we're in the filter and just completed a filter expr, - # "and" as a junction should just be ignored. - elif in_filter and just_filtered and element[1] == "and": - just_filtered = False - continue - # Otherwise we're in a filter and this term is a key name - elif curr_key is None: - curr_key = element[1] - continue - # Otherwise we're an operator or the key value - elif curr_key is not None: - # I think = is the only possible thing to support with PathElem syntax as is - if element[0] in [">", "<"]: - raise Exception("Only = supported as filter operand!") - if element[0] == "=": - continue - else: - # We have a full key here, put it in the map - if curr_key in curr_elem.key.keys(): - raise Exception("Key already in key map!") - curr_elem.key[curr_key] = element[0].strip("'\"") - curr_key = None - just_filtered = True - # Keys/filters in general should be totally cleaned up at this point. - if curr_key: - raise Exception("Hanging key filter! Incomplete XPath?") - # If we have a dangling element that hasn't been completed due to no - # / element then let's just append the final element. - if curr_elem: - path_elems.append(curr_elem) - curr_elem = None - if any([curr_elem, curr_key, in_filter]): - raise Exception("Unfinished elements in XPath parsing!") - path.elem.extend(path_elems) - return path - - def combine_configs(self, payload, last_xpath, cfg): - """Walking from end to finish 2 xpaths merge so combine them - |--config - |---last xpath config--| - ----| |--config - | - | pick these up --> |--config - |---this xpath config--| - |--config - Parameters - ---------- - payload: dict of partial payload - last_xpath: last xpath that was processed - xpath: colliding xpath - config: dict of values associated to colliding xpath - """ - xpath, config, is_key = cfg - lp = last_xpath.split('/') - xp = xpath.split('/') - base = [] - top = '' - for i, seg in enumerate(zip(lp, xp)): - if seg[0] != seg[1]: - top = seg[1] - break - base = '/' + '/'.join(xp[i:]) - cfg = (base, config, False) - extended_payload = {top: self.xpath_to_json([cfg])} - payload.update(extended_payload) - return payload - - def xpath_to_json(self, configs, last_xpath='', payload={}): - """Try to combine Xpaths/values into a common payload (recursive). - - Parameters - ---------- - configs: tuple of xpath/value dict - last_xpath: str of last xpath that was recusivly processed. - payload: dict being recursively built for JSON transformation. - - Returns - ------- - dict of combined xpath/value dict. - """ - for i, cfg in enumerate(configs, 1): - xpath, config, is_key = cfg - if last_xpath and xpath not in last_xpath: - # Branched config here |---config - # |---last xpath config--| - # --| |---config - # |---this xpath config - payload = self.combine_configs(payload, last_xpath, cfg) - return self.xpath_to_json(configs[i:], xpath, payload) - xpath_segs = xpath.split('/') - xpath_segs.reverse() - for seg in xpath_segs: - if not seg: - continue - if payload: - if is_key: - if seg in payload: - if isinstance(payload[seg], list): - payload[seg].append(config) - elif isinstance(payload[seg], dict): - payload[seg].update(config) - else: - payload.update(config) - payload = {seg: [payload]} - else: - config.update(payload) - payload = {seg: config} - return self.xpath_to_json(configs[i:], xpath, payload) - else: - if is_key: - payload = {seg: [config]} - else: - payload = {seg: config} - return self.xpath_to_json(configs[i:], xpath, payload) - return payload - - # Pattern to detect keys in an xpath - RE_FIND_KEYS = re.compile(r'\[.*?\]') - - def get_payload(self, configs): - """Common Xpaths were detected so try to consolidate them. - - Parameter - --------- - configs: tuple of xpath/value dicts - """ - # Number of updates are limited so try to consolidate into lists. - xpaths_cfg = [] - first_key = set() - # Find first common keys for all xpaths_cfg of collection. - for config in configs: - xpath = next(iter(config.keys())) - - # Change configs to tuples (xpath, config) for easier management - xpaths_cfg.append((xpath, config[xpath])) - - xpath_split = xpath.split('/') - for seg in xpath_split: - if '[' in seg: - first_key.add(seg) - break - - # Common first key/configs represents one GNMI update - updates = [] - for key in first_key: - update = [] - remove_cfg = [] - for config in xpaths_cfg: - xpath, cfg = config - if key in xpath: - update.append(config) - else: - for k, v in cfg.items(): - if '[{0}="{1}"]'.format(k, v) not in key: - break - else: - # This cfg sets the first key so we don't need it - remove_cfg.append((xpath, cfg)) - if update: - for upd in update: - # Remove this config out of main list - xpaths_cfg.remove(upd) - for rem_cfg in remove_cfg: - # Sets a key in update path so remove it - xpaths_cfg.remove(rem_cfg) - updates.append(update) - break - - # Add remaining configs to updates - if xpaths_cfg: - updates.append(xpaths_cfg) - - # Combine all xpath configs of each update if possible - xpaths = [] - compressed_updates = [] - for update in updates: - xpath_consolidated = {} - config_compressed = [] - for seg in update: - xpath, config = seg - if xpath in xpath_consolidated: - xpath_consolidated[xpath].update(config) - else: - xpath_consolidated[xpath] = config - config_compressed.append((xpath, xpath_consolidated[xpath])) - xpaths.append(xpath) - - # Now get the update path for this batch of configs - common_xpath = os.path.commonprefix(xpaths) - cfg_compressed = [] - keys = [] - - # Need to reverse the configs to build the dict correctly - config_compressed.reverse() - for seg in config_compressed: - is_key = False - prepend_path = '' - xpath, config = seg - end_path = xpath[len(common_xpath):] - if end_path.startswith('['): - # Don't start payload with a list - tmp = common_xpath.split('/') - prepend_path = '/' + tmp.pop() - common_xpath = '/'.join(tmp) - end_path = prepend_path + end_path - - # Building json, need to identify configs that set keys - for key in keys: - if [k for k in config.keys() if k in key]: - is_key = True - keys += re.findall(self.RE_FIND_KEYS, end_path) - cfg_compressed.append((end_path, config, is_key)) - - update = (common_xpath, cfg_compressed) - compressed_updates.append(update) - - updates = [] - for update in compressed_updates: - common_xpath, cfgs = update - payload = self.xpath_to_json(cfgs) - updates.append( - ( - common_xpath, - json.dumps(payload).encode('utf-8') - ) - ) - return updates - - def xml_path_to_path_elem(self, request): - """Convert XML Path Language 1.0 Xpath to gNMI Path/PathElement. - - Modeled after YANG/NETCONF Xpaths. - - References: - * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths - * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev - * https://tools.ietf.org/html/rfc6020#section-6.4 - * https://tools.ietf.org/html/rfc6020#section-9.13 - * https://tools.ietf.org/html/rfc6241 - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: DME, device, or openconfig - """ - - paths = [] - message = { - 'update': [], - 'replace': [], - 'delete': [], - 'get': [], - } - if 'nodes' not in request: - # TODO: raw rpc? - return paths - else: - namespace_modules = {} - origin = 'DME' - for prefix, nspace in request.get('namespace', {}).items(): - if '/Cisco-IOS-' in nspace: - module = nspace[nspace.rfind('/') + 1:] - elif '/cisco-nx' in nspace: # NXOS lowercases namespace - module = 'Cisco-NX-OS-device' - elif '/openconfig.net' in nspace: - module = 'openconfig-' - module += nspace[nspace.rfind('/') + 1:] - elif 'urn:ietf:params:xml:ns:yang:' in nspace: - module = nspace.replace( - 'urn:ietf:params:xml:ns:yang:', '') - if module: - namespace_modules[prefix] = module - - for node in request.get('nodes', []): - if 'xpath' not in node: - log.error('Xpath is not in message') - else: - xpath = node['xpath'] - value = node.get('value', '') - edit_op = node.get('edit-op', '') - - for pfx, ns in namespace_modules.items(): - # NXOS does not support prefixes yet so clear them out - if pfx in xpath and 'openconfig' in ns: - origin = 'openconfig' - xpath = xpath.replace(pfx + ':', '') - if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - elif pfx in xpath and 'device' in ns: - origin = 'device' - xpath = xpath.replace(pfx + ':', '') - if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - if edit_op: - if edit_op in ['create', 'merge', 'replace']: - xpath_lst = xpath.split('/') - name = xpath_lst.pop() - xpath = '/'.join(xpath_lst) - if edit_op == 'replace': - if not message['replace']: - message['replace'] = [{ - xpath: {name: value} - }] - else: - message['replace'].append( - {xpath: {name: value}} - ) - else: - if not message['update']: - message['update'] = [{ - xpath: {name: value} - }] - else: - message['update'].append( - {xpath: {name: value}} - ) - elif edit_op in ['delete', 'remove']: - if message['delete']: - message['delete'].add(xpath) - else: - message['delete'] = set(xpath) - else: - message['get'].append(xpath) - return namespace_modules, message, origin - - -if __name__ == '__main__': - from pprint import pprint as pp - import grpc - from cisco_gnmi.auth import CiscoAuthPlugin - channel = grpc.secure_channel( - '127.0.0.1:9339', - grpc.composite_channel_credentials( - grpc.ssl_channel_credentials(), - grpc.metadata_call_credentials( - CiscoAuthPlugin( - 'admin', - 'its_a_secret' - ) - ) - ) - ) - client = Client(channel) - request = { - 'namespace': { - 'oc-acl': 'http://openconfig.net/yang/acl' - }, - 'nodes': [ - { - 'value': 'testacl', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', - 'edit-op': 'merge' - }, - { - 'value': 'ACL_IPV4', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', - 'edit-op': 'merge' - }, - { - 'value': '10', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', - 'edit-op': 'merge' - }, - { - 'value': '20.20.20.1/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', - 'edit-op': 'merge' - }, - { - 'value': 'IP_TCP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', - 'edit-op': 'merge' - }, - { - 'value': '10.10.10.10/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', - 'edit-op': 'merge' - }, - { - 'value': 'DROP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', - 'edit-op': 'merge' - } - ] - } - modules, message, origin = client.xml_path_to_path_elem(request) - pp(modules) - pp(message) - pp(origin) - """ - # Expected output - ================= - {'oc-acl': 'openconfig-acl'} - {'delete': [], - 'get': [], - 'replace': [], - 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, - {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}]} - 'openconfig' - """ - # Feed converted XML Path Language 1.0 Xpaths to create updates - updates = client.create_updates(message['update'], origin) - pp(updates) - """ - # Expected output - ================= - [path { - origin: "openconfig" - elem { - name: "acl" - } - elem { - name: "acl-sets" - } - elem { - name: "acl-set" - key { - key: "name" - value: "testacl" - } - key { - key: "type" - value: "ACL_IPV4" - } - } - elem { - name: "acl-entries" - } - } - val { - json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" - } - ] - # update is now ready to be sent through gNMI SetRequest - """ diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index f18610b..e09386f 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -29,7 +29,9 @@ import os from six import string_types -from .client import Client, proto, util +from cisco_gnmi import proto, util +from cisco_gnmi.client import Client +from cisco_gnmi.xpath_util import parse_xpath_to_gnmi_path logger = logging.getLogger(__name__) @@ -85,7 +87,7 @@ def delete_xpaths(self, xpaths, prefix=None): xpath = "{prefix}{xpath}".format(prefix=prefix, xpath=xpath) else: xpath = "{prefix}/{xpath}".format(prefix=prefix, xpath=xpath) - paths.append(self.parse_xpath_to_gnmi_path(xpath)) + paths.append(parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) def set_json(self, update_json_configs=None, replace_json_configs=None, @@ -158,9 +160,9 @@ def get_xpaths(self, xpaths, data_type="ALL", if isinstance(xpaths, (list, set)): gnmi_path = [] for xpath in set(xpaths): - gnmi_path.append(self.parse_xpath_to_gnmi_path(xpath, origin)) + gnmi_path.append(parse_xpath_to_gnmi_path(xpath, origin)) elif isinstance(xpaths, string_types): - gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths, origin)] + gnmi_path = [parse_xpath_to_gnmi_path(xpaths, origin)] else: raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" @@ -239,7 +241,7 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path( + parse_xpath_to_gnmi_path( xpath_subscription, origin ) @@ -253,7 +255,7 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path( + path = parse_xpath_to_gnmi_path( xpath_subscription["path"], origin ) @@ -282,46 +284,3 @@ def subscribe_xpaths( 15 * '=', str(subscription_list)) ) return self.subscribe([subscription_list]) - - def parse_xpath_to_gnmi_path(self, xpath, origin): - """Origin defaults to YANG (device) paths - Otherwise specify "DME" as origin - """ - if origin is None: - if any(map(xpath.startswith, ["/Cisco-NX-OS-device", "/ietf-interfaces"])): - origin = "device" - else: - origin = "DME" - - return super(NXClient, self).parse_xpath_to_gnmi_path(xpath, origin) - - def xpath_to_path_elem(self, request): - """Convert XML Path Language 1.0 formed xpath to gNMI PathElement. - - Modeled after NETCONF Xpaths RFC 6020 (See client.py for use example). - - References: - * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths - * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev - * https://tools.ietf.org/html/rfc6020#section-6.4 - * https://tools.ietf.org/html/rfc6020#section-9.13 - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: DME, device, or openconfig - """ - return super(NXClient, self).xml_path_to_path_elem(request) diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index c7775c4..e552139 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -28,7 +28,9 @@ import os from six import string_types -from .client import Client, proto, util +from cisco_gnmi import proto, util +from cisco_gnmi.client import Client +from cisco_gnmi.xpath_util import parse_xpath_to_gnmi_path logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -108,7 +110,7 @@ def delete_xpaths(self, xpaths, prefix=None): xpath = "{prefix}{xpath}".format(prefix=prefix, xpath=xpath) else: xpath = "{prefix}/{xpath}".format(prefix=prefix, xpath=xpath) - paths.append(self.parse_xpath_to_gnmi_path(xpath)) + paths.append(parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) def set_json(self, update_json_configs=None, replace_json_configs=None, @@ -180,9 +182,9 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None) if isinstance(xpaths, (list, set)): gnmi_path = [] for xpath in set(xpaths): - gnmi_path.append(self.parse_xpath_to_gnmi_path(xpath, origin)) + gnmi_path.append(parse_xpath_to_gnmi_path(xpath, origin)) elif isinstance(xpaths, string_types): - gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths, origin)] + gnmi_path = [parse_xpath_to_gnmi_path(xpaths, origin)] else: raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" @@ -261,7 +263,7 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path( + parse_xpath_to_gnmi_path( xpath_subscription, origin ) @@ -275,7 +277,7 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path( + path = parse_xpath_to_gnmi_path( xpath_subscription["path"], origin ) @@ -304,47 +306,3 @@ def subscribe_xpaths( 15 * '=', str(subscription_list)) ) return self.subscribe([subscription_list]) - - def parse_xpath_to_gnmi_path(self, xpath, origin=None): - """Naively tries to intelligently (non-sequitur!) origin - Otherwise assume rfc7951 - legacy is not considered - """ - if origin is None: - # naive but effective - if ":" in xpath: - origin = "openconfig" - else: - origin = "rfc7951" - return super(XEClient, self).parse_xpath_to_gnmi_path(xpath, origin) - - def xpath_to_path_elem(self, request): - """Convert XML Path Language 1.0 formed xpath to gNMI PathElement. - - Modeled after NETCONF Xpaths RFC 6020. - - References: - * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths - * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev - * https://tools.ietf.org/html/rfc6020#section-6.4 - * https://tools.ietf.org/html/rfc6020#section-9.13 - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: DME, device, or openconfig - """ - return super(XEClient, self).xml_path_to_path_elem(request) diff --git a/src/cisco_gnmi/xpath_util.py b/src/cisco_gnmi/xpath_util.py new file mode 100644 index 0000000..51525de --- /dev/null +++ b/src/cisco_gnmi/xpath_util.py @@ -0,0 +1,516 @@ +import os +import re +import json +from xml.etree.ElementPath import xpath_tokenizer_re +from six import string_types +from cisco_gnmi import proto + + +def parse_xpath_to_gnmi_path(xpath, origin=None): + """Parses an XPath to proto.gnmi_pb2.Path. + This function should be overridden by any child classes for origin logic. + + Effectively wraps the std XML XPath tokenizer and traverses + the identified groups. Parsing robustness needs to be validated. + Probably best to formalize as a state machine sometime. + TODO: Formalize tokenizer traversal via state machine. + """ + if not isinstance(xpath, string_types): + raise Exception("xpath must be a string!") + path = proto.gnmi_pb2.Path() + if origin: + if not isinstance(origin, string_types): + raise Exception("origin must be a string!") + path.origin = origin + curr_elem = proto.gnmi_pb2.PathElem() + in_filter = False + just_filtered = False + curr_key = None + # TODO: Lazy + xpath = xpath.strip("/") + xpath_elements = xpath_tokenizer_re.findall(xpath) + path_elems = [] + for index, element in enumerate(xpath_elements): + # stripped initial /, so this indicates a completed element + if element[0] == "/": + if not curr_elem.name: + raise Exception( + "Current PathElem has no name yet is trying to be pushed to path! Invalid XPath?" + ) + path_elems.append(curr_elem) + curr_elem = proto.gnmi_pb2.PathElem() + continue + # We are entering a filter + elif element[0] == "[": + in_filter = True + continue + # We are exiting a filter + elif element[0] == "]": + in_filter = False + continue + # If we're not in a filter then we're a PathElem name + elif not in_filter: + curr_elem.name = element[1] + # Skip blank spaces + elif not any([element[0], element[1]]): + continue + # If we're in the filter and just completed a filter expr, + # "and" as a junction should just be ignored. + elif in_filter and just_filtered and element[1] == "and": + just_filtered = False + continue + # Otherwise we're in a filter and this term is a key name + elif curr_key is None: + curr_key = element[1] + continue + # Otherwise we're an operator or the key value + elif curr_key is not None: + # I think = is the only possible thing to support with PathElem syntax as is + if element[0] in [">", "<"]: + raise Exception("Only = supported as filter operand!") + if element[0] == "=": + continue + else: + # We have a full key here, put it in the map + if curr_key in curr_elem.key.keys(): + raise Exception("Key already in key map!") + curr_elem.key[curr_key] = element[0].strip("'\"") + curr_key = None + just_filtered = True + # Keys/filters in general should be totally cleaned up at this point. + if curr_key: + raise Exception("Hanging key filter! Incomplete XPath?") + # If we have a dangling element that hasn't been completed due to no + # / element then let's just append the final element. + if curr_elem: + path_elems.append(curr_elem) + curr_elem = None + if any([curr_elem, curr_key, in_filter]): + raise Exception("Unfinished elements in XPath parsing!") + path.elem.extend(path_elems) + return path + + +def combine_configs(payload, last_xpath, cfg): + """Walking from end to finish, 2 xpaths merge, so combine them. + |--config + |---last xpath config--| + ----| |--config + | + | pick these up --> |--config + |---this xpath config--| + |--config + Parameters + ---------- + payload: dict of partial payload + last_xpath: last xpath that was processed + xpath: colliding xpath + config: dict of values associated to colliding xpath + """ + xpath, config, is_key = cfg + lp = last_xpath.split('/') + xp = xpath.split('/') + base = [] + top = '' + for i, seg in enumerate(zip(lp, xp)): + if seg[0] != seg[1]: + top = seg[1] + break + base = '/' + '/'.join(xp[i:]) + cfg = (base, config, False) + extended_payload = {top: xpath_to_json([cfg])} + payload.update(extended_payload) + return payload + + +def xpath_to_json(configs, last_xpath='', payload={}): + """Try to combine Xpaths/values into a common payload (recursive). + + Parameters + ---------- + configs: tuple of xpath/value dict + last_xpath: str of last xpath that was recusivly processed. + payload: dict being recursively built for JSON transformation. + + Returns + ------- + dict of combined xpath/value dict. + """ + for i, cfg in enumerate(configs, 1): + xpath, config, is_key = cfg + if last_xpath and xpath not in last_xpath: + # Branched config here |---config + # |---last xpath config--| + # --| |---config + # |---this xpath config + payload = combine_configs(payload, last_xpath, cfg) + return xpath_to_json(configs[i:], xpath, payload) + xpath_segs = xpath.split('/') + xpath_segs.reverse() + for seg in xpath_segs: + if not seg: + continue + if payload: + if is_key: + if seg in payload: + if isinstance(payload[seg], list): + payload[seg].append(config) + elif isinstance(payload[seg], dict): + payload[seg].update(config) + else: + payload.update(config) + payload = {seg: [payload]} + else: + config.update(payload) + payload = {seg: config} + return xpath_to_json(configs[i:], xpath, payload) + else: + if is_key: + payload = {seg: [config]} + else: + payload = {seg: config} + return xpath_to_json(configs[i:], xpath, payload) + return payload + + +# Pattern to detect keys in an xpath +RE_FIND_KEYS = re.compile(r'\[.*?\]') + + +def get_payload(configs): + """Common Xpaths were detected so try to consolidate them. + + Parameter + --------- + configs: tuple of xpath/value dicts + """ + # Number of updates are limited so try to consolidate into lists. + xpaths_cfg = [] + first_key = set() + # Find first common keys for all xpaths_cfg of collection. + for config in configs: + xpath = next(iter(config.keys())) + + # Change configs to tuples (xpath, config) for easier management + xpaths_cfg.append((xpath, config[xpath])) + + xpath_split = xpath.split('/') + for seg in xpath_split: + if '[' in seg: + first_key.add(seg) + break + + # Common first key/configs represents one GNMI update + updates = [] + for key in first_key: + update = [] + remove_cfg = [] + for config in xpaths_cfg: + xpath, cfg = config + if key in xpath: + update.append(config) + else: + for k, v in cfg.items(): + if '[{0}="{1}"]'.format(k, v) not in key: + break + else: + # This cfg sets the first key so we don't need it + remove_cfg.append((xpath, cfg)) + if update: + for upd in update: + # Remove this config out of main list + xpaths_cfg.remove(upd) + for rem_cfg in remove_cfg: + # Sets a key in update path so remove it + xpaths_cfg.remove(rem_cfg) + updates.append(update) + break + + # Add remaining configs to updates + if xpaths_cfg: + updates.append(xpaths_cfg) + + # Combine all xpath configs of each update if possible + xpaths = [] + compressed_updates = [] + for update in updates: + xpath_consolidated = {} + config_compressed = [] + for seg in update: + xpath, config = seg + if xpath in xpath_consolidated: + xpath_consolidated[xpath].update(config) + else: + xpath_consolidated[xpath] = config + config_compressed.append((xpath, xpath_consolidated[xpath])) + xpaths.append(xpath) + + # Now get the update path for this batch of configs + common_xpath = os.path.commonprefix(xpaths) + cfg_compressed = [] + keys = [] + + # Need to reverse the configs to build the dict correctly + config_compressed.reverse() + for seg in config_compressed: + is_key = False + prepend_path = '' + xpath, config = seg + end_path = xpath[len(common_xpath):] + if end_path.startswith('['): + # Don't start payload with a list + tmp = common_xpath.split('/') + prepend_path = '/' + tmp.pop() + common_xpath = '/'.join(tmp) + end_path = prepend_path + end_path + + # Building json, need to identify configs that set keys + for key in keys: + if [k for k in config.keys() if k in key]: + is_key = True + keys += re.findall(RE_FIND_KEYS, end_path) + cfg_compressed.append((end_path, config, is_key)) + + update = (common_xpath, cfg_compressed) + compressed_updates.append(update) + + updates = [] + for update in compressed_updates: + common_xpath, cfgs = update + payload = xpath_to_json(cfgs) + updates.append( + ( + common_xpath, + json.dumps(payload).encode('utf-8') + ) + ) + return updates + + +def xml_path_to_path_elem(request): + """Convert XML Path Language 1.0 Xpath to gNMI Path/PathElement. + + Modeled after YANG/NETCONF Xpaths. + + References: + * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths + * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev + * https://tools.ietf.org/html/rfc6020#section-6.4 + * https://tools.ietf.org/html/rfc6020#section-9.13 + * https://tools.ietf.org/html/rfc6241 + + Parameters + --------- + request: dict containing request namespace and nodes to be worked on. + namespace: dict of : + nodes: list of dict + : Xpath pointing to resource + : value to set resource to + : equivelant NETCONF edit-config operation + + Returns + ------- + tuple: namespace_modules, message dict, origin + namespace_modules: dict of : + Needed for future support. + message dict: 4 lists containing possible updates, replaces, + deletes, or gets derived form input nodes. + origin str: DME, device, or openconfig + """ + + paths = [] + message = { + 'update': [], + 'replace': [], + 'delete': [], + 'get': [], + } + if 'nodes' not in request: + # TODO: raw rpc? + return paths + else: + namespace_modules = {} + origin = 'DME' + for prefix, nspace in request.get('namespace', {}).items(): + if '/Cisco-IOS-' in nspace: + module = nspace[nspace.rfind('/') + 1:] + elif '/cisco-nx' in nspace: # NXOS lowercases namespace + module = 'Cisco-NX-OS-device' + elif '/openconfig.net' in nspace: + module = 'openconfig-' + module += nspace[nspace.rfind('/') + 1:] + elif 'urn:ietf:params:xml:ns:yang:' in nspace: + module = nspace.replace( + 'urn:ietf:params:xml:ns:yang:', '') + if module: + namespace_modules[prefix] = module + + for node in request.get('nodes', []): + if 'xpath' not in node: + log.error('Xpath is not in message') + else: + xpath = node['xpath'] + value = node.get('value', '') + edit_op = node.get('edit-op', '') + + for pfx, ns in namespace_modules.items(): + # NXOS does not support prefixes yet so clear them out + if pfx in xpath and 'openconfig' in ns: + origin = 'openconfig' + xpath = xpath.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') + elif pfx in xpath and 'device' in ns: + origin = 'device' + xpath = xpath.replace(pfx + ':', '') + if isinstance(value, string_types): + value = value.replace(pfx + ':', '') + if edit_op: + if edit_op in ['create', 'merge', 'replace']: + xpath_lst = xpath.split('/') + name = xpath_lst.pop() + xpath = '/'.join(xpath_lst) + if edit_op == 'replace': + if not message['replace']: + message['replace'] = [{ + xpath: {name: value} + }] + else: + message['replace'].append( + {xpath: {name: value}} + ) + else: + if not message['update']: + message['update'] = [{ + xpath: {name: value} + }] + else: + message['update'].append( + {xpath: {name: value}} + ) + elif edit_op in ['delete', 'remove']: + if message['delete']: + message['delete'].add(xpath) + else: + message['delete'] = set(xpath) + else: + message['get'].append(xpath) + return namespace_modules, message, origin + + +if __name__ == '__main__': + from pprint import pprint as pp + import grpc + from cisco_gnmi.auth import CiscoAuthPlugin + from cisco_gnmi.client import Client + + channel = grpc.secure_channel( + '127.0.0.1:9339', + grpc.composite_channel_credentials( + grpc.ssl_channel_credentials(), + grpc.metadata_call_credentials( + CiscoAuthPlugin( + 'admin', + 'its_a_secret' + ) + ) + ) + ) + client = Client(channel) + request = { + 'namespace': { + 'oc-acl': 'http://openconfig.net/yang/acl' + }, + 'nodes': [ + { + 'value': 'testacl', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', + 'edit-op': 'merge' + }, + { + 'value': 'ACL_IPV4', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', + 'edit-op': 'merge' + }, + { + 'value': '10', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', + 'edit-op': 'merge' + }, + { + 'value': '20.20.20.1/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', + 'edit-op': 'merge' + }, + { + 'value': 'IP_TCP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', + 'edit-op': 'merge' + }, + { + 'value': '10.10.10.10/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', + 'edit-op': 'merge' + }, + { + 'value': 'DROP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', + 'edit-op': 'merge' + } + ] + } + modules, message, origin = xml_path_to_path_elem(request) + pp(modules) + pp(message) + pp(origin) + """ + # Expected output + ================= + {'oc-acl': 'openconfig-acl'} + {'delete': [], + 'get': [], + 'replace': [], + 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, + {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}]} + 'openconfig' + """ + # Feed converted XML Path Language 1.0 Xpaths to create updates + updates = client.create_updates(message['update'], origin) + pp(updates) + """ + # Expected output + ================= + [path { + origin: "openconfig" + elem { + name: "acl" + } + elem { + name: "acl-sets" + } + elem { + name: "acl-set" + key { + key: "name" + value: "testacl" + } + key { + key: "type" + value: "ACL_IPV4" + } + } + elem { + name: "acl-entries" + } + } + val { + json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" + } + ] + # update is now ready to be sent through gNMI SetRequest + """ From 5e8bafd889efc17c9026061a739170974d9c14a2 Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 13 Apr 2020 16:44:47 -0700 Subject: [PATCH 15/27] Added a couple sanity tests --- src/cisco_gnmi/xpath_util.py | 4 ++ tests/test_xpath.py | 117 +++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 tests/test_xpath.py diff --git a/src/cisco_gnmi/xpath_util.py b/src/cisco_gnmi/xpath_util.py index 51525de..8ba6b30 100644 --- a/src/cisco_gnmi/xpath_util.py +++ b/src/cisco_gnmi/xpath_util.py @@ -1,11 +1,15 @@ import os import re import json +import logger from xml.etree.ElementPath import xpath_tokenizer_re from six import string_types from cisco_gnmi import proto +log = logging.getLogger(__name__) + + def parse_xpath_to_gnmi_path(xpath, origin=None): """Parses an XPath to proto.gnmi_pb2.Path. This function should be overridden by any child classes for origin logic. diff --git a/tests/test_xpath.py b/tests/test_xpath.py new file mode 100644 index 0000000..e6f342d --- /dev/null +++ b/tests/test_xpath.py @@ -0,0 +1,117 @@ +import pytest + +from cisco_gnmi import xpath_util + + +def test_parse_xpath_to_gnmi_path(xpath, origin=None): + result = xpath_util.parse_xpath_to_gnmi_path( + PARSE_XPATH_TO_GNMI, + origin='openconfig' + ) + assert result == GNMI_UPDATE + + +def test_combine_configs(payload, last_xpath, cfg): + pass + + +def test_xpath_to_json(configs, last_xpath='', payload={}): + pass + + +def test_get_payload(configs): + pass + + +def test_xml_path_to_path_elem(request): + result = xpath_util.xml_path_to_path_elem(XML_PATH_LANGUAGE_1) + assert result == PARSE_XPATH_TO_GNMI + + +XML_PATH_LANGUAGE_1 = { + 'namespace': { + 'oc-acl': 'http://openconfig.net/yang/acl' + }, + 'nodes': [ + { + 'value': 'testacl', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', + 'edit-op': 'merge' + }, + { + 'value': 'ACL_IPV4', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', + 'edit-op': 'merge' + }, + { + 'value': '10', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', + 'edit-op': 'merge' + }, + { + 'value': '20.20.20.1/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', + 'edit-op': 'merge' + }, + { + 'value': 'IP_TCP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', + 'edit-op': 'merge' + }, + { + 'value': '10.10.10.10/32', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', + 'edit-op': 'merge' + }, + { + 'value': 'DROP', + 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', + 'edit-op': 'merge' + } + ] +} + + +PARSE_XPATH_TO_GNMI = { + 'delete': [], + 'get': [], + 'replace': [], + 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, + {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}] +} + + +GNMI_UPDATE = """ +[path { +origin: "openconfig" +elem { + name: "acl" +} +elem { + name: "acl-sets" +} +elem { + name: "acl-set" + key { + key: "name" + value: "testacl" + } + key { + key: "type" + value: "ACL_IPV4" + } +} +elem { + name: "acl-entries" +} +} +val { +json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" +} +] +""" From 919ae7e953ece952b7f4c14c23219ce54fa96fcd Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 13 Apr 2020 17:14:45 -0700 Subject: [PATCH 16/27] Fix test import --- tests/test_xpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_xpath.py b/tests/test_xpath.py index e6f342d..2fe67be 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -1,6 +1,6 @@ import pytest -from cisco_gnmi import xpath_util +from src.cisco_gnmi import xpath_util def test_parse_xpath_to_gnmi_path(xpath, origin=None): From b20017a66cdd10396cf44926ac2eb7ecd09bc818 Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 13 Apr 2020 17:19:16 -0700 Subject: [PATCH 17/27] Added tests for xpath utilities. --- tests/test_xpath.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_xpath.py b/tests/test_xpath.py index 2fe67be..7d706b3 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -1,5 +1,3 @@ -import pytest - from src.cisco_gnmi import xpath_util From ec45b3c23c741d204a537d1129759c903aea7096 Mon Sep 17 00:00:00 2001 From: miott Date: Tue, 14 Apr 2020 08:04:50 -0700 Subject: [PATCH 18/27] Added sanity tests for xpath_util. --- Makefile | 2 +- src/cisco_gnmi/client.py | 16 +-- src/cisco_gnmi/nx.py | 47 ++++---- src/cisco_gnmi/xe.py | 44 ++++---- src/cisco_gnmi/xpath_util.py | 209 ++++++++++++++++------------------- tests/test_xpath.py | 98 ++++++++++++---- 6 files changed, 216 insertions(+), 200 deletions(-) diff --git a/Makefile b/Makefile index 1829293..01430e5 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ mostlyclean: ## Runs tests. .PHONY: test test: - pipenv run pytest $(TEST_DIR) -v -s --disable-warnings + pipenv run pytest $(TEST_DIR) -vv -s --disable-warnings ## Creates coverage report. .PHONY: coverage diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 796b3f1..2dc5d9f 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -31,9 +31,9 @@ import os from six import string_types -from cisco_gnmi import proto -from cisco_gnmi import util -from cisco_gnmi.xpath_util import get_payload, parse_xpath_to_gnmi_path +from . import proto +from . import util +from .xpath_util import get_payload, parse_xpath_to_gnmi_path class Client(object): @@ -268,9 +268,7 @@ def check_configs(self, configs): logger.debug("Handling already serialized JSON object.") configs = [configs] elif not isinstance(configs, (list, set)): - raise Exception( - "{0} must be an iterable of configs!".format(str(configs)) - ) + raise Exception("{0} must be an iterable of configs!".format(str(configs))) return configs def create_updates(self, configs, origin, json_ietf=False): @@ -310,11 +308,7 @@ def create_updates(self, configs, origin, json_ietf=False): for update_cfg in update_configs: xpath, payload = update_cfg update = proto.gnmi_pb2.Update() - update.path.CopyFrom( - parse_xpath_to_gnmi_path( - xpath, origin=origin - ) - ) + update.path.CopyFrom(parse_xpath_to_gnmi_path(xpath, origin=origin)) if json_ietf: update.val.json_ietf_val = payload else: diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index e09386f..78e036e 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -29,9 +29,9 @@ import os from six import string_types -from cisco_gnmi import proto, util -from cisco_gnmi.client import Client -from cisco_gnmi.xpath_util import parse_xpath_to_gnmi_path +from . import proto, util +from .client import Client +from .xpath_util import parse_xpath_to_gnmi_path logger = logging.getLogger(__name__) @@ -58,6 +58,7 @@ class NXClient(Client): >>> capabilities = client.capabilities() >>> print(capabilities) """ + def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. @@ -90,8 +91,13 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json(self, update_json_configs=None, replace_json_configs=None, - origin='device', json_ietf=False): + def set_json( + self, + update_json_configs=None, + replace_json_configs=None, + origin="device", + json_ietf=False, + ): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -114,22 +120,17 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, raise Exception("Must supply at least one set of configurations to method!") updates = self.create_updates( - update_json_configs, - origin=origin, - json_ietf=json_ietf + update_json_configs, origin=origin, json_ietf=json_ietf ) replaces = self.create_updates( - replace_json_configs, - origin=origin, - json_ietf=json_ietf + replace_json_configs, origin=origin, json_ietf=json_ietf ) for update in updates + replaces: - logger.debug('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) + logger.debug("\nGNMI set:\n{0}\n{1}".format(9 * "=", str(update))) return self.set(updates=updates, replaces=replaces) - def get_xpaths(self, xpaths, data_type="ALL", - encoding="JSON", origin='openconfig'): + def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON", origin="openconfig"): """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. Parameters @@ -167,7 +168,7 @@ def get_xpaths(self, xpaths, data_type="ALL", raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) - logger.debug('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) + logger.debug("GNMI get:\n{0}\n{1}".format(9 * "=", str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -177,7 +178,7 @@ def subscribe_xpaths( sub_mode="SAMPLE", encoding="PROTO", sample_interval=Client._NS_IN_S * 10, - origin='openconfig' + origin="openconfig", ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -241,10 +242,7 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - parse_xpath_to_gnmi_path( - xpath_subscription, - origin - ) + parse_xpath_to_gnmi_path(xpath_subscription, origin) ) subscription.mode = util.validate_proto_enum( "sub_mode", @@ -255,10 +253,7 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = parse_xpath_to_gnmi_path( - xpath_subscription["path"], - origin - ) + path = parse_xpath_to_gnmi_path(xpath_subscription["path"], origin) arg_dict = { "path": path, "mode": sub_mode, @@ -280,7 +275,7 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) - logger.debug('GNMI subscribe:\n{0}\n{1}'.format( - 15 * '=', str(subscription_list)) + logger.debug( + "GNMI subscribe:\n{0}\n{1}".format(15 * "=", str(subscription_list)) ) return self.subscribe([subscription_list]) diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index e552139..0fa5e20 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -28,9 +28,9 @@ import os from six import string_types -from cisco_gnmi import proto, util -from cisco_gnmi.client import Client -from cisco_gnmi.xpath_util import parse_xpath_to_gnmi_path +from . import proto, util +from .client import Client +from .xpath_util import parse_xpath_to_gnmi_path logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -81,6 +81,7 @@ class XEClient(Client): ... >>> delete_response = client.delete_xpaths('/Cisco-IOS-XE-native:native/hostname') """ + def delete_xpaths(self, xpaths, prefix=None): """A convenience wrapper for set() which constructs Paths from supplied xpaths to be passed to set() as the delete parameter. @@ -113,8 +114,13 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json(self, update_json_configs=None, replace_json_configs=None, - origin='device', json_ietf=True): + def set_json( + self, + update_json_configs=None, + replace_json_configs=None, + origin="device", + json_ietf=True, + ): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -137,17 +143,13 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, raise Exception("Must supply at least one set of configurations to method!") updates = self.create_updates( - update_json_configs, - origin=origin, - json_ietf=json_ietf + update_json_configs, origin=origin, json_ietf=json_ietf ) replaces = self.create_updates( - replace_json_configs, - origin=origin, - json_ietf=json_ietf + replace_json_configs, origin=origin, json_ietf=json_ietf ) for update in updates + replaces: - logger.debug('\nGNMI set:\n{0}\n{1}'.format(9 * '=', str(update))) + logger.debug("\nGNMI set:\n{0}\n{1}".format(9 * "=", str(update))) return self.set(updates=updates, replaces=replaces) @@ -189,7 +191,7 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None) raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) - logger.debug('GNMI get:\n{0}\n{1}'.format(9 * '=', str(gnmi_path))) + logger.debug("GNMI get:\n{0}\n{1}".format(9 * "=", str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -199,7 +201,7 @@ def subscribe_xpaths( sub_mode="SAMPLE", encoding="JSON_IETF", sample_interval=Client._NS_IN_S * 10, - origin='openconfig' + origin="openconfig", ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -263,10 +265,7 @@ def subscribe_xpaths( if isinstance(xpath_subscription, string_types): subscription = proto.gnmi_pb2.Subscription() subscription.path.CopyFrom( - parse_xpath_to_gnmi_path( - xpath_subscription, - origin - ) + parse_xpath_to_gnmi_path(xpath_subscription, origin) ) subscription.mode = util.validate_proto_enum( "sub_mode", @@ -277,10 +276,7 @@ def subscribe_xpaths( ) subscription.sample_interval = sample_interval elif isinstance(xpath_subscription, dict): - path = parse_xpath_to_gnmi_path( - xpath_subscription["path"], - origin - ) + path = parse_xpath_to_gnmi_path(xpath_subscription["path"], origin) arg_dict = { "path": path, "mode": sub_mode, @@ -302,7 +298,7 @@ def subscribe_xpaths( raise Exception("xpath in list must be xpath or dict/Path!") subscriptions.append(subscription) subscription_list.subscription.extend(subscriptions) - logger.debug('GNMI subscribe:\n{0}\n{1}'.format( - 15 * '=', str(subscription_list)) + logger.debug( + "GNMI subscribe:\n{0}\n{1}".format(15 * "=", str(subscription_list)) ) return self.subscribe([subscription_list]) diff --git a/src/cisco_gnmi/xpath_util.py b/src/cisco_gnmi/xpath_util.py index 8ba6b30..698a901 100644 --- a/src/cisco_gnmi/xpath_util.py +++ b/src/cisco_gnmi/xpath_util.py @@ -1,10 +1,10 @@ import os import re import json -import logger +import logging from xml.etree.ElementPath import xpath_tokenizer_re from six import string_types -from cisco_gnmi import proto +from . import proto log = logging.getLogger(__name__) @@ -112,22 +112,22 @@ def combine_configs(payload, last_xpath, cfg): config: dict of values associated to colliding xpath """ xpath, config, is_key = cfg - lp = last_xpath.split('/') - xp = xpath.split('/') + lp = last_xpath.split("/") + xp = xpath.split("/") base = [] - top = '' + top = "" for i, seg in enumerate(zip(lp, xp)): if seg[0] != seg[1]: top = seg[1] break - base = '/' + '/'.join(xp[i:]) + base = "/" + "/".join(xp[i:]) cfg = (base, config, False) extended_payload = {top: xpath_to_json([cfg])} payload.update(extended_payload) return payload -def xpath_to_json(configs, last_xpath='', payload={}): +def xpath_to_json(configs, last_xpath="", payload={}): """Try to combine Xpaths/values into a common payload (recursive). Parameters @@ -149,7 +149,7 @@ def xpath_to_json(configs, last_xpath='', payload={}): # |---this xpath config payload = combine_configs(payload, last_xpath, cfg) return xpath_to_json(configs[i:], xpath, payload) - xpath_segs = xpath.split('/') + xpath_segs = xpath.split("/") xpath_segs.reverse() for seg in xpath_segs: if not seg: @@ -178,7 +178,7 @@ def xpath_to_json(configs, last_xpath='', payload={}): # Pattern to detect keys in an xpath -RE_FIND_KEYS = re.compile(r'\[.*?\]') +RE_FIND_KEYS = re.compile(r"\[.*?\]") def get_payload(configs): @@ -186,7 +186,7 @@ def get_payload(configs): Parameter --------- - configs: tuple of xpath/value dicts + configs: list of {xpath: {name: value}} dicts """ # Number of updates are limited so try to consolidate into lists. xpaths_cfg = [] @@ -198,9 +198,9 @@ def get_payload(configs): # Change configs to tuples (xpath, config) for easier management xpaths_cfg.append((xpath, config[xpath])) - xpath_split = xpath.split('/') + xpath_split = xpath.split("/") for seg in xpath_split: - if '[' in seg: + if "[" in seg: first_key.add(seg) break @@ -258,14 +258,14 @@ def get_payload(configs): config_compressed.reverse() for seg in config_compressed: is_key = False - prepend_path = '' + prepend_path = "" xpath, config = seg - end_path = xpath[len(common_xpath):] - if end_path.startswith('['): + end_path = xpath[len(common_xpath) :] + if end_path.startswith("["): # Don't start payload with a list - tmp = common_xpath.split('/') - prepend_path = '/' + tmp.pop() - common_xpath = '/'.join(tmp) + tmp = common_xpath.split("/") + prepend_path = "/" + tmp.pop() + common_xpath = "/".join(tmp) end_path = prepend_path + end_path # Building json, need to identify configs that set keys @@ -282,12 +282,7 @@ def get_payload(configs): for update in compressed_updates: common_xpath, cfgs = update payload = xpath_to_json(cfgs) - updates.append( - ( - common_xpath, - json.dumps(payload).encode('utf-8') - ) - ) + updates.append((common_xpath, json.dumps(payload).encode("utf-8"))) return updates @@ -324,144 +319,128 @@ def xml_path_to_path_elem(request): paths = [] message = { - 'update': [], - 'replace': [], - 'delete': [], - 'get': [], + "update": [], + "replace": [], + "delete": [], + "get": [], } - if 'nodes' not in request: + if "nodes" not in request: # TODO: raw rpc? return paths else: namespace_modules = {} - origin = 'DME' - for prefix, nspace in request.get('namespace', {}).items(): - if '/Cisco-IOS-' in nspace: - module = nspace[nspace.rfind('/') + 1:] - elif '/cisco-nx' in nspace: # NXOS lowercases namespace - module = 'Cisco-NX-OS-device' - elif '/openconfig.net' in nspace: - module = 'openconfig-' - module += nspace[nspace.rfind('/') + 1:] - elif 'urn:ietf:params:xml:ns:yang:' in nspace: - module = nspace.replace( - 'urn:ietf:params:xml:ns:yang:', '') + origin = "DME" + for prefix, nspace in request.get("namespace", {}).items(): + if "/Cisco-IOS-" in nspace: + module = nspace[nspace.rfind("/") + 1 :] + elif "/cisco-nx" in nspace: # NXOS lowercases namespace + module = "Cisco-NX-OS-device" + elif "/openconfig.net" in nspace: + module = "openconfig-" + module += nspace[nspace.rfind("/") + 1 :] + elif "urn:ietf:params:xml:ns:yang:" in nspace: + module = nspace.replace("urn:ietf:params:xml:ns:yang:", "") if module: namespace_modules[prefix] = module - for node in request.get('nodes', []): - if 'xpath' not in node: - log.error('Xpath is not in message') + for node in request.get("nodes", []): + if "xpath" not in node: + log.error("Xpath is not in message") else: - xpath = node['xpath'] - value = node.get('value', '') - edit_op = node.get('edit-op', '') + xpath = node["xpath"] + value = node.get("value", "") + edit_op = node.get("edit-op", "") for pfx, ns in namespace_modules.items(): # NXOS does not support prefixes yet so clear them out - if pfx in xpath and 'openconfig' in ns: - origin = 'openconfig' - xpath = xpath.replace(pfx + ':', '') + if pfx in xpath and "openconfig" in ns: + origin = "openconfig" + xpath = xpath.replace(pfx + ":", "") if isinstance(value, string_types): - value = value.replace(pfx + ':', '') - elif pfx in xpath and 'device' in ns: - origin = 'device' - xpath = xpath.replace(pfx + ':', '') + value = value.replace(pfx + ":", "") + elif pfx in xpath and "device" in ns: + origin = "device" + xpath = xpath.replace(pfx + ":", "") if isinstance(value, string_types): - value = value.replace(pfx + ':', '') + value = value.replace(pfx + ":", "") if edit_op: - if edit_op in ['create', 'merge', 'replace']: - xpath_lst = xpath.split('/') + if edit_op in ["create", "merge", "replace"]: + xpath_lst = xpath.split("/") name = xpath_lst.pop() - xpath = '/'.join(xpath_lst) - if edit_op == 'replace': - if not message['replace']: - message['replace'] = [{ - xpath: {name: value} - }] + xpath = "/".join(xpath_lst) + if edit_op == "replace": + if not message["replace"]: + message["replace"] = [{xpath: {name: value}}] else: - message['replace'].append( - {xpath: {name: value}} - ) + message["replace"].append({xpath: {name: value}}) else: - if not message['update']: - message['update'] = [{ - xpath: {name: value} - }] + if not message["update"]: + message["update"] = [{xpath: {name: value}}] else: - message['update'].append( - {xpath: {name: value}} - ) - elif edit_op in ['delete', 'remove']: - if message['delete']: - message['delete'].add(xpath) + message["update"].append({xpath: {name: value}}) + elif edit_op in ["delete", "remove"]: + if message["delete"]: + message["delete"].add(xpath) else: - message['delete'] = set(xpath) + message["delete"] = set(xpath) else: - message['get'].append(xpath) + message["get"].append(xpath) return namespace_modules, message, origin -if __name__ == '__main__': +if __name__ == "__main__": from pprint import pprint as pp import grpc from cisco_gnmi.auth import CiscoAuthPlugin from cisco_gnmi.client import Client channel = grpc.secure_channel( - '127.0.0.1:9339', + "127.0.0.1:9339", grpc.composite_channel_credentials( grpc.ssl_channel_credentials(), - grpc.metadata_call_credentials( - CiscoAuthPlugin( - 'admin', - 'its_a_secret' - ) - ) - ) + grpc.metadata_call_credentials(CiscoAuthPlugin("admin", "its_a_secret")), + ), ) client = Client(channel) request = { - 'namespace': { - 'oc-acl': 'http://openconfig.net/yang/acl' - }, - 'nodes': [ + "namespace": {"oc-acl": "http://openconfig.net/yang/acl"}, + "nodes": [ { - 'value': 'testacl', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', - 'edit-op': 'merge' + "value": "testacl", + "xpath": "/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name", + "edit-op": "merge", }, { - 'value': 'ACL_IPV4', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', - 'edit-op': 'merge' + "value": "ACL_IPV4", + "xpath": "/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type", + "edit-op": "merge", }, { - 'value': '10', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', - 'edit-op': 'merge' + "value": "10", + "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', + "edit-op": "merge", }, { - 'value': '20.20.20.1/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', - 'edit-op': 'merge' + "value": "20.20.20.1/32", + "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', + "edit-op": "merge", }, { - 'value': 'IP_TCP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', - 'edit-op': 'merge' + "value": "IP_TCP", + "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', + "edit-op": "merge", }, { - 'value': '10.10.10.10/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', - 'edit-op': 'merge' + "value": "10.10.10.10/32", + "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', + "edit-op": "merge", }, { - 'value': 'DROP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', - 'edit-op': 'merge' - } - ] + "value": "DROP", + "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', + "edit-op": "merge", + }, + ], } modules, message, origin = xml_path_to_path_elem(request) pp(modules) @@ -484,7 +463,7 @@ def xml_path_to_path_elem(request): 'openconfig' """ # Feed converted XML Path Language 1.0 Xpaths to create updates - updates = client.create_updates(message['update'], origin) + updates = client.create_updates(message["update"], origin) pp(updates) """ # Expected output diff --git a/tests/test_xpath.py b/tests/test_xpath.py index 7d706b3..6fd556f 100644 --- a/tests/test_xpath.py +++ b/tests/test_xpath.py @@ -1,29 +1,63 @@ +import json from src.cisco_gnmi import xpath_util -def test_parse_xpath_to_gnmi_path(xpath, origin=None): +def test_parse_xpath_to_gnmi_path(): result = xpath_util.parse_xpath_to_gnmi_path( - PARSE_XPATH_TO_GNMI, + '/acl/acl-sets/acl-set', origin='openconfig' ) - assert result == GNMI_UPDATE + assert str(result) == GNMI_UPDATE_ACL_SET -def test_combine_configs(payload, last_xpath, cfg): +def test_combine_configs(): pass -def test_xpath_to_json(configs, last_xpath='', payload={}): +def test_xpath_to_json(): pass -def test_get_payload(configs): - pass +def test_get_payload(): + result = xpath_util.get_payload(PARSE_XPATH_TO_GNMI[1]['update']) + xpath, config = result[0] + assert xpath == '/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries' + # json_val + cfg = json.loads(config) + assert cfg == { + 'acl-entry': [ + { + 'config': {'forwarding-action': 'DROP'}, + 'ipv4': {'config': {'destination-address': '20.20.20.1/32', + 'protocol': 'IP_TCP', + 'source-address': '10.10.10.10/32'} + }, + 'sequence-id': '10' + } + ] + } -def test_xml_path_to_path_elem(request): +def test_xml_path_to_path_elem(): result = xpath_util.xml_path_to_path_elem(XML_PATH_LANGUAGE_1) - assert result == PARSE_XPATH_TO_GNMI + assert result == ( + {'oc-acl': 'openconfig-acl'}, # module + { # config + 'delete': [], + 'get': [], + 'replace': [], + 'update': [ + {'/acl/acl-sets/acl-set': {'name': 'testacl'}}, + {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}} + ] + }, + 'openconfig' # origin + ) XML_PATH_LANGUAGE_1 = { @@ -70,22 +104,27 @@ def test_xml_path_to_path_elem(request): } -PARSE_XPATH_TO_GNMI = { - 'delete': [], - 'get': [], - 'replace': [], - 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, - {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}] -} +PARSE_XPATH_TO_GNMI = ( + {'oc-acl': 'openconfig-acl'}, # module + { # config + 'delete': [], + 'get': [], + 'replace': [], + 'update': [ + {'/acl/acl-sets/acl-set': {'name': 'testacl'}}, + {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, + {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}} + ] + }, + 'openconfig' # origin +) -GNMI_UPDATE = """ -[path { +GNMI_UPDATE_ACL_ENTRY = """[path { origin: "openconfig" elem { name: "acl" @@ -113,3 +152,16 @@ def test_xml_path_to_path_elem(request): } ] """ + + +GNMI_UPDATE_ACL_SET = """origin: "openconfig" +elem { + name: "acl" +} +elem { + name: "acl-sets" +} +elem { + name: "acl-set" +} +""" From 878204dc144e275bbbc9a66ddaa513cf8a110d08 Mon Sep 17 00:00:00 2001 From: miott Date: Tue, 14 Apr 2020 13:29:45 -0700 Subject: [PATCH 19/27] Fixed grpc insecure channel connect --- src/cisco_gnmi/builder.py | 41 ++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/cisco_gnmi/builder.py b/src/cisco_gnmi/builder.py index db1d0d6..513c57d 100644 --- a/src/cisco_gnmi/builder.py +++ b/src/cisco_gnmi/builder.py @@ -271,9 +271,11 @@ def construct(self): channel_ssl_creds = None channel_metadata_creds = None channel_creds = None - channel_ssl_creds = grpc.ssl_channel_credentials( - self.__root_certificates, self.__private_key, self.__certificate_chain - ) + channel_ssl_creds = None + if any((self.__root_certificates, self.__private_key, self.__certificate_chain)): + channel_ssl_creds = grpc.ssl_channel_credentials( + self.__root_certificates, self.__private_key, self.__certificate_chain + ) if self.__username and self.__password: channel_metadata_creds = grpc.metadata_call_credentials( CiscoAuthPlugin(self.__username, self.__password) @@ -284,25 +286,28 @@ def construct(self): channel_ssl_creds, channel_metadata_creds ) logging.debug("Using SSL/metadata authentication composite credentials.") - else: + elif channel_ssl_creds: channel_creds = channel_ssl_creds logging.debug("Using SSL credentials, no metadata authentication.") - if self.__ssl_target_name_override is not False: - if self.__ssl_target_name_override is None: - if not self.__root_certificates: - raise Exception("Deriving override requires root certificate!") - self.__ssl_target_name_override = get_cn_from_cert( - self.__root_certificates + if channel_creds: + if self.__ssl_target_name_override is not False: + if self.__ssl_target_name_override is None: + if not self.__root_certificates: + raise Exception("Deriving override requires root certificate!") + self.__ssl_target_name_override = get_cn_from_cert( + self.__root_certificates + ) + logging.warning( + "Overriding SSL option from certificate could increase MITM susceptibility!" + ) + self.set_channel_option( + "grpc.ssl_target_name_override", self.__ssl_target_name_override ) - logging.warning( - "Overriding SSL option from certificate could increase MITM susceptibility!" - ) - self.set_channel_option( - "grpc.ssl_target_name_override", self.__ssl_target_name_override + channel = grpc.secure_channel( + self.__target_netloc.netloc, channel_creds, self.__channel_options ) - channel = grpc.secure_channel( - self.__target_netloc.netloc, channel_creds, self.__channel_options - ) + else: + channel = grpc.insecure_channel(self.__target_netloc.netloc) if self.__client_class is None: self.set_os() client = self.__client_class(channel) From 06d98d904054e6686fe466027ca96c9bc9f3c6bf Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 27 Apr 2020 16:40:23 -0700 Subject: [PATCH 20/27] Reseting to cisco_gnmi/master. --- Makefile | 4 +- README.md | 398 +++++++++++++++++++++++++++- scripts/gen_certs.sh | 51 +++- setup.py | 25 ++ src/cisco_gnmi/__init__.py | 3 +- src/cisco_gnmi/builder.py | 45 ++-- src/cisco_gnmi/client.py | 275 ++++++++++++++----- src/cisco_gnmi/nx.py | 225 ++++------------ src/cisco_gnmi/util.py | 14 +- src/cisco_gnmi/xe.py | 187 ++++++------- src/cisco_gnmi/xpath_util.py | 499 ----------------------------------- src/cisco_gnmi/xr.py | 85 +++--- 12 files changed, 889 insertions(+), 922 deletions(-) delete mode 100644 src/cisco_gnmi/xpath_util.py diff --git a/Makefile b/Makefile index 01430e5..4ffc67c 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ mostlyclean: ## Runs tests. .PHONY: test test: - pipenv run pytest $(TEST_DIR) -vv -s --disable-warnings + pipenv run pytest $(TEST_DIR) -v -s --disable-warnings ## Creates coverage report. .PHONY: coverage @@ -84,4 +84,4 @@ help: helpMessage = ""; \ } \ }' \ - $(MAKEFILE_LIST) + $(MAKEFILE_LIST) \ No newline at end of file diff --git a/README.md b/README.md index 27980e4..c0f6dfb 100644 --- a/README.md +++ b/README.md @@ -4,16 +4,53 @@ This library wraps gNMI functionality to ease usage with Cisco implementations in Python programs. Derived from [openconfig/gnmi](https://github.com/openconfig/gnmi/tree/master/proto). +- [cisco-gnmi-python](#cisco-gnmi-python) + - [Usage](#usage) + - [cisco-gnmi CLI](#cisco-gnmi-cli) + - [ClientBuilder](#clientbuilder) + - [Initialization Examples](#initialization-examples) + - [Client](#client) + - [NXClient](#nxclient) + - [XEClient](#xeclient) + - [XRClient](#xrclient) + - [gNMI](#gnmi) + - [Development](#development) + - [Get Source](#get-source) + - [Code Hygiene](#code-hygiene) + - [Recompile Protobufs](#recompile-protobufs) + - [CLI Usage](#cli-usage) + - [Capabilities](#capabilities) + - [Usage](#usage-1) + - [Output](#output) + - [Get](#get) + - [Usage](#usage-2) + - [Output](#output-1) + - [Set](#set) + - [Usage](#usage-3) + - [Output](#output-2) + - [Subscribe](#subscribe) + - [Usage](#usage-4) + - [Output](#output-3) + - [Licensing](#licensing) + - [Issues](#issues) + - [Related Projects](#related-projects) + ## Usage ```bash pip install cisco-gnmi python -c "import cisco_gnmi; print(cisco_gnmi)" +cisco-gnmi --help ``` -This library covers the gNMI defined `capabilities`, `get`, `set`, and `subscribe` RPCs, and helper clients provide OS-specific recommendations. As commonalities and differences are identified this library will be refactored as necessary. +This library covers the gNMI defined `Capabilities`, `Get`, `Set`, and `Subscribe` RPCs, and helper clients provide OS-specific recommendations. A CLI (`cisco-gnmi`) is also available upon installation. As commonalities and differences are identified between OS functionality this library will be refactored as necessary. + +Several examples of library usage are available in [`examples/`](examples/). The `cisco-gnmi` CLI script found at [`src/cisco_gnmi/cli.py`](src/cisco_gnmi/cli.py) may also be useful. It is *highly* recommended that users of the library learn [Google Protocol Buffers](https://developers.google.com/protocol-buffers/) syntax to significantly ease usage. Understanding how to read Protocol Buffers, and reference [`gnmi.proto`](https://github.com/openconfig/gnmi/blob/master/proto/gnmi/gnmi.proto), will be immensely useful for utilizing gNMI and any other gRPC interface. +### cisco-gnmi CLI +Since `v1.0.5` a gNMI CLI is available as `cisco-gnmi` when this module is installed. `Capabilities`, `Get`, rudimentary `Set`, and `Subscribe` are supported. The CLI may be useful for simply interacting with a Cisco gNMI service, and also serves as a reference for how to use this `cisco_gnmi` library. CLI usage is documented at the bottom of this README in [CLI Usage](#cli-usage). + ### ClientBuilder Since `v1.0.0` a builder pattern is available with `ClientBuilder`. `ClientBuilder` provides several `set_*` methods which define the intended `Client` connectivity and a `construct` method to construct and return the desired `Client`. There are several major methods involved here: @@ -181,6 +218,365 @@ If a new `gnmi.proto` definition is released, use `update_protos.sh` to recompil ./update_protos.sh ``` +## CLI Usage +The below details the current `cisco-gnmi` usage options. Please note that `Set` operations may be destructive to operations and should be tested in lab conditions. + +``` +cisco-gnmi --help +usage: +cisco-gnmi [] + +Supported RPCs: +capabilities +subscribe +get +set + +cisco-gnmi capabilities 127.0.0.1:57500 +cisco-gnmi get 127.0.0.1:57500 +cisco-gnmi set 127.0.0.1:57500 -delete_xpath Cisco-IOS-XR-shellutil-cfg:host-names/host-name +cisco-gnmi subscribe 127.0.0.1:57500 -debug -auto_ssl_target_override -dump_file intfcounters.proto.txt + +See --help for RPC options. + + +gNMI CLI demonstrating library usage. + +positional arguments: + rpc gNMI RPC to perform against network element. + +optional arguments: + -h, --help show this help message and exit +``` + +### Capabilities +This command will output the `CapabilitiesResponse` to `stdout`. +``` +cisco-gnmi capabilities 127.0.0.1:57500 -auto_ssl_target_override +``` + +#### Usage +``` +cisco-gnmi capabilities --help +usage: cisco-gnmi [-h] [-os {None,IOS XR,NX-OS,IOS XE}] + [-root_certificates ROOT_CERTIFICATES] + [-private_key PRIVATE_KEY] + [-certificate_chain CERTIFICATE_CHAIN] + [-ssl_target_override SSL_TARGET_OVERRIDE] + [-auto_ssl_target_override] [-debug] + netloc + +Performs Capabilities RPC against network element. + +positional arguments: + netloc : + +optional arguments: + -h, --help show this help message and exit + -os {None,IOS XR,NX-OS,IOS XE} + OS wrapper to utilize. Defaults to IOS XR. + -root_certificates ROOT_CERTIFICATES + Root certificates for secure connection. + -private_key PRIVATE_KEY + Private key for secure connection. + -certificate_chain CERTIFICATE_CHAIN + Certificate chain for secure connection. + -ssl_target_override SSL_TARGET_OVERRIDE + gRPC SSL target override option. + -auto_ssl_target_override + Use root_certificates first CN as + grpc.ssl_target_name_override. + -debug Print debug messages. +``` + +#### Output +``` +[cisco-gnmi-python] cisco-gnmi capabilities redacted:57500 -auto_ssl_target_override +Username: admin +Password: +WARNING:root:Overriding SSL option from certificate could increase MITM susceptibility! +INFO:root:supported_models { + name: "Cisco-IOS-XR-qos-ma-oper" + organization: "Cisco Systems, Inc." + version: "2019-04-05" +} +... +``` + +### Get +This command will output the `GetResponse` to `stdout`. `-xpath` may be specified multiple times to specify multiple `Path`s for the `GetRequest`. +``` +cisco-gnmi get 127.0.0.1:57500 -os "IOS XR" -xpath /interfaces/interface/state/counters -auto_ssl_target_override +``` + +#### Usage +``` +cisco-gnmi get --help +usage: cisco-gnmi [-h] [-xpath XPATH] + [-encoding {JSON,BYTES,PROTO,ASCII,JSON_IETF}] + [-data_type {ALL,CONFIG,STATE,OPERATIONAL}] [-dump_json] + [-os {None,IOS XR,NX-OS,IOS XE}] + [-root_certificates ROOT_CERTIFICATES] + [-private_key PRIVATE_KEY] + [-certificate_chain CERTIFICATE_CHAIN] + [-ssl_target_override SSL_TARGET_OVERRIDE] + [-auto_ssl_target_override] [-debug] + netloc + +Performs Get RPC against network element. + +positional arguments: + netloc : + +optional arguments: + -h, --help show this help message and exit + -xpath XPATH XPaths to Get. + -encoding {JSON,BYTES,PROTO,ASCII,JSON_IETF} + gNMI Encoding. + -data_type {ALL,CONFIG,STATE,OPERATIONAL} + gNMI GetRequest DataType + -dump_json Dump as JSON instead of textual protos. + -os {None,IOS XR,NX-OS,IOS XE} + OS wrapper to utilize. Defaults to IOS XR. + -root_certificates ROOT_CERTIFICATES + Root certificates for secure connection. + -private_key PRIVATE_KEY + Private key for secure connection. + -certificate_chain CERTIFICATE_CHAIN + Certificate chain for secure connection. + -ssl_target_override SSL_TARGET_OVERRIDE + gRPC SSL target override option. + -auto_ssl_target_override + Use root_certificates first CN as + grpc.ssl_target_name_override. + -debug Print debug messages. +``` + +#### Output +``` +[cisco-gnmi-python] cisco-gnmi get redacted:57500 -os "IOS XR" -xpath /interfaces/interface/state/counters -auto_ssl_target_override +Username: admin +Password: +WARNING:root:Overriding SSL option from certificate could increase MITM susceptibility! +INFO:root:notification { + timestamp: 1585607100869287743 + update { + path { + elem { + name: "interfaces" + } + elem { + name: "interface" + } + elem { + name: "state" + } + elem { + name: "counters" + } + } + val { + json_ietf_val: "{\"in-unicast-pkts\":\"0\",\"in-octets\":\"0\"... +``` + +### Set +Please note that `Set` operations may be destructive to operations and should be tested in lab conditions. Behavior is not fully validated. + +#### Usage +``` +cisco-gnmi set --help +usage: cisco-gnmi [-h] [-update_json_config UPDATE_JSON_CONFIG] + [-replace_json_config REPLACE_JSON_CONFIG] + [-delete_xpath DELETE_XPATH] [-no_ietf] [-dump_json] + [-os {None,IOS XR,NX-OS,IOS XE}] + [-root_certificates ROOT_CERTIFICATES] + [-private_key PRIVATE_KEY] + [-certificate_chain CERTIFICATE_CHAIN] + [-ssl_target_override SSL_TARGET_OVERRIDE] + [-auto_ssl_target_override] [-debug] + netloc + +Performs Set RPC against network element. + +positional arguments: + netloc : + +optional arguments: + -h, --help show this help message and exit + -update_json_config UPDATE_JSON_CONFIG + JSON-modeled config to apply as an update. + -replace_json_config REPLACE_JSON_CONFIG + JSON-modeled config to apply as a replace. + -delete_xpath DELETE_XPATH + XPaths to delete. + -no_ietf JSON is not IETF conformant. + -dump_json Dump as JSON instead of textual protos. + -os {None,IOS XR,NX-OS,IOS XE} + OS wrapper to utilize. Defaults to IOS XR. + -root_certificates ROOT_CERTIFICATES + Root certificates for secure connection. + -private_key PRIVATE_KEY + Private key for secure connection. + -certificate_chain CERTIFICATE_CHAIN + Certificate chain for secure connection. + -ssl_target_override SSL_TARGET_OVERRIDE + gRPC SSL target override option. + -auto_ssl_target_override + Use root_certificates first CN as + grpc.ssl_target_name_override. + -debug Print debug messages. +``` + +#### Output +Let's create a harmless loopback interface based from [`openconfig-interfaces.yang`](https://github.com/openconfig/public/blob/master/release/models/interfaces/openconfig-interfaces.yang). + +`config.json` +```json +{ + "openconfig-interfaces:interfaces": { + "interface": [ + { + "name": "Loopback9339" + } + ] + } +} +``` + +``` +[cisco-gnmi-python] cisco-gnmi set redacted:57500 -os "IOS XR" -auto_ssl_target_override -update_json_config config.json +Username: admin +Password: +WARNING:root:Overriding SSL option from certificate could increase MITM susceptibility! +INFO:root:response { + path { + origin: "openconfig-interfaces" + elem { + name: "interfaces" + } + } + message { + } + op: UPDATE +} +message { +} +timestamp: 1585715036783451369 +``` + +And on IOS XR...a loopback interface! +``` +... +interface Loopback9339 +! +... +``` + +### Subscribe +This command will output the `SubscribeResponse` to `stdout` or `-dump_file`. `-xpath` may be specified multiple times to specify multiple `Path`s for the `GetRequest`. + +``` +cisco-gnmi subscribe 127.0.0.1:57500 -os "IOS XR" -xpath /interfaces/interface/state/counters -auto_ssl_target_override +``` + +#### Usage +``` +cisco-gnmi subscribe --help +usage: cisco-gnmi [-h] [-xpath XPATH] [-interval INTERVAL] + [-mode {TARGET_DEFINED,ON_CHANGE,SAMPLE}] + [-suppress_redundant] + [-heartbeat_interval HEARTBEAT_INTERVAL] + [-dump_file DUMP_FILE] [-dump_json] [-sync_stop] + [-sync_start] [-encoding {JSON,BYTES,PROTO,ASCII,JSON_IETF}] + [-os {None,IOS XR,NX-OS,IOS XE}] + [-root_certificates ROOT_CERTIFICATES] + [-private_key PRIVATE_KEY] + [-certificate_chain CERTIFICATE_CHAIN] + [-ssl_target_override SSL_TARGET_OVERRIDE] + [-auto_ssl_target_override] [-debug] + netloc + +Performs Subscribe RPC against network element. + +positional arguments: + netloc : + +optional arguments: + -h, --help show this help message and exit + -xpath XPATH XPath to subscribe to. + -interval INTERVAL Sample interval in seconds for Subscription. Defaults + to 10. + -mode {TARGET_DEFINED,ON_CHANGE,SAMPLE} + SubscriptionMode for Subscription. Defaults to SAMPLE. + -suppress_redundant Suppress redundant information in Subscription. + -heartbeat_interval HEARTBEAT_INTERVAL + Heartbeat interval in seconds. + -dump_file DUMP_FILE Filename to dump to. Defaults to stdout. + -dump_json Dump as JSON instead of textual protos. + -sync_stop Stop on sync_response. + -sync_start Start processing messages after sync_response. + -encoding {JSON,BYTES,PROTO,ASCII,JSON_IETF} + gNMI Encoding. Defaults to whatever Client wrapper + prefers. + -os {None,IOS XR,NX-OS,IOS XE} + OS wrapper to utilize. Defaults to IOS XR. + -root_certificates ROOT_CERTIFICATES + Root certificates for secure connection. + -private_key PRIVATE_KEY + Private key for secure connection. + -certificate_chain CERTIFICATE_CHAIN + Certificate chain for secure connection. + -ssl_target_override SSL_TARGET_OVERRIDE + gRPC SSL target override option. + -auto_ssl_target_override + Use root_certificates first CN as + grpc.ssl_target_name_override. + -debug Print debug messages. +``` + +#### Output +``` +[cisco-gnmi-python] cisco-gnmi subscribe redacted:57500 -os "IOS XR" -xpath /interfaces/interface/state/counters -auto_ssl_target_override +Username: admin +Password: +WARNING:root:Overriding SSL option from certificate could increase MITM susceptibility! +INFO:root:Dumping responses to stdout as textual proto ... +INFO:root:Subscribing to: +/interfaces/interface/state/counters +INFO:root:update { + timestamp: 1585607768601000000 + prefix { + origin: "openconfig" + elem { + name: "interfaces" + } + elem { + name: "interface" + key { + key: "name" + value: "Null0" + } + } + elem { + name: "state" + } + elem { + name: "counters" + } + } + update { + path { + elem { + name: "in-octets" + } + } + val { + uint_val: 0 + } + } +... +``` + ## Licensing `cisco-gnmi-python` is licensed as [Apache License, Version 2.0](LICENSE). diff --git a/scripts/gen_certs.sh b/scripts/gen_certs.sh index a15f10f..4c20bc5 100755 --- a/scripts/gen_certs.sh +++ b/scripts/gen_certs.sh @@ -3,30 +3,55 @@ CERT_BASE="certs" -if [ -z $1 ]; then - echo "Usage: gen_certs.sh []" +if [ -z $1 ] || [ -z $2 ]; then + echo "Usage: gen_certs.sh []" exit 1 fi +server_hostname=$1 +ip=$2 +password=$3 + mkdir -p $CERT_BASE +function print_red () { + printf "\033[0;31m$1 ...\033[0m\n" +} + # Setting up a CA -openssl genrsa -out $CERT_BASE/rootCA.key 2048 -openssl req -subj /C=/ST=/L=/O=/CN=rootCA -x509 -new -nodes -key $CERT_BASE/rootCA.key -sha256 -out $CERT_BASE/rootCA.pem +if [ -f "$CERT_BASE/rootCA.key" ] && [ -f "$CERT_BASE/rootCA.pem" ]; then + print_red "SKIPPING rootCA generation, already exist" +else + print_red "GENERATING rootCA" + openssl genrsa -out $CERT_BASE/rootCA.key 2048 + openssl req -subj /C=/ST=/L=/O=/CN=rootCA -x509 -new -nodes -key $CERT_BASE/rootCA.key -sha256 -days 1095 -out $CERT_BASE/rootCA.pem +fi # Setting up device cert and key +print_red "GENERATING device certificates with CN $server_hostname and IP $ip" openssl genrsa -out $CERT_BASE/device.key 2048 -openssl req -subj /C=/ST=/L=/O=/CN=$1 -new -key $CERT_BASE/device.key -out $CERT_BASE/device.csr -openssl x509 -req -in $CERT_BASE/device.csr -CA $CERT_BASE/rootCA.pem -CAkey $CERT_BASE/rootCA.key -CAcreateserial -out $CERT_BASE/device.crt -sha256 +openssl req -subj /C=/ST=/L=/O=/CN=$server_hostname -new -key $CERT_BASE/device.key -out $CERT_BASE/device.csr +openssl x509 -req -in $CERT_BASE/device.csr -CA $CERT_BASE/rootCA.pem -CAkey $CERT_BASE/rootCA.key -CAcreateserial -out $CERT_BASE/device.crt -days 1095 -sha256 -extfile <(printf "%s" "subjectAltName=DNS:$server_hostname,IP:$ip") -# Encrypt device key - needed for input to IOS -if [ ! -z $2 ]; then - openssl rsa -des3 -in $CERT_BASE/device.key -out $CERT_BASE/device.des3.key -passout pass:$2 +# Encrypt device key +if [ ! -z $password ]; then + print_red "ENCRYPTING device certificates and bundling with password" + # DES 3 for device, needed for input to IOS XE + openssl rsa -des3 -in $CERT_BASE/device.key -out $CERT_BASE/device.des3.key -passout pass:$password + # PKCS #12 for device, needed for NX-OS + # Uncertain if this is correct + openssl pkcs12 -export -out $CERT_BASE/device.pfx -inkey $CERT_BASE/device.key -in $CERT_BASE/device.crt -certfile $CERT_BASE/rootCA.pem -password pass:$password else - echo "Skipping device key encryption." + print_red "SKIPPING device key encryption" fi # Setting up client cert and key -openssl genrsa -out $CERT_BASE/client.key 2048 -openssl req -subj /C=/ST=/L=/O=/CN=gnmi_client -new -key $CERT_BASE/client.key -out $CERT_BASE/client.csr -openssl x509 -req -in $CERT_BASE/client.csr -CA $CERT_BASE/rootCA.pem -CAkey $CERT_BASE/rootCA.key -CAcreateserial -out $CERT_BASE/client.crt -sha256 \ No newline at end of file +if [ -f "$CERT_BASE/client.key" ] && [ -f "$CERT_BASE/client.crt" ]; then + print_red "SKIPPING client certificates generation, already exist" +else + hostname=$(hostname) + print_red "GENERATING client certificates with CN $hostname" + openssl genrsa -out $CERT_BASE/client.key 2048 + openssl req -subj /C=/ST=/L=/O=/CN=$hostname -new -key $CERT_BASE/client.key -out $CERT_BASE/client.csr + openssl x509 -req -in $CERT_BASE/client.csr -CA $CERT_BASE/rootCA.pem -CAkey $CERT_BASE/rootCA.key -CAcreateserial -out $CERT_BASE/client.crt -days 1095 -sha256 +fi diff --git a/setup.py b/setup.py index 08867d5..31e3303 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,27 @@ +#!/usr/bin/env python +"""Copyright 2020 Cisco Systems +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +The contents of this file are licensed under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with the +License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations under +the License. +""" + """Derived from Flask https://github.com/pallets/flask/blob/master/setup.py """ @@ -69,4 +93,5 @@ "coverage", ], }, + entry_points={"console_scripts": ["cisco-gnmi = cisco_gnmi.cli:main"]}, ) diff --git a/src/cisco_gnmi/__init__.py b/src/cisco_gnmi/__init__.py index a0c51f8..60f494f 100644 --- a/src/cisco_gnmi/__init__.py +++ b/src/cisco_gnmi/__init__.py @@ -29,6 +29,5 @@ from .nx import NXClient from .xe import XEClient from .builder import ClientBuilder -from . import xpath_util -__version__ = "1.0.4" +__version__ = "1.0.7" diff --git a/src/cisco_gnmi/builder.py b/src/cisco_gnmi/builder.py index 513c57d..a74a219 100644 --- a/src/cisco_gnmi/builder.py +++ b/src/cisco_gnmi/builder.py @@ -76,9 +76,13 @@ class ClientBuilder(object): os_class_map = { None: Client, + "None": Client, "IOS XR": XRClient, + "XR": XRClient, "NX-OS": NXClient, + "NX": NXClient, "IOS XE": XEClient, + "XE": XEClient, } def __init__(self, target): @@ -271,11 +275,9 @@ def construct(self): channel_ssl_creds = None channel_metadata_creds = None channel_creds = None - channel_ssl_creds = None - if any((self.__root_certificates, self.__private_key, self.__certificate_chain)): - channel_ssl_creds = grpc.ssl_channel_credentials( - self.__root_certificates, self.__private_key, self.__certificate_chain - ) + channel_ssl_creds = grpc.ssl_channel_credentials( + self.__root_certificates, self.__private_key, self.__certificate_chain + ) if self.__username and self.__password: channel_metadata_creds = grpc.metadata_call_credentials( CiscoAuthPlugin(self.__username, self.__password) @@ -286,28 +288,25 @@ def construct(self): channel_ssl_creds, channel_metadata_creds ) logging.debug("Using SSL/metadata authentication composite credentials.") - elif channel_ssl_creds: + else: channel_creds = channel_ssl_creds logging.debug("Using SSL credentials, no metadata authentication.") - if channel_creds: - if self.__ssl_target_name_override is not False: - if self.__ssl_target_name_override is None: - if not self.__root_certificates: - raise Exception("Deriving override requires root certificate!") - self.__ssl_target_name_override = get_cn_from_cert( - self.__root_certificates - ) - logging.warning( - "Overriding SSL option from certificate could increase MITM susceptibility!" - ) - self.set_channel_option( - "grpc.ssl_target_name_override", self.__ssl_target_name_override + if self.__ssl_target_name_override is not False: + if self.__ssl_target_name_override is None: + if not self.__root_certificates: + raise Exception("Deriving override requires root certificate!") + self.__ssl_target_name_override = get_cn_from_cert( + self.__root_certificates ) - channel = grpc.secure_channel( - self.__target_netloc.netloc, channel_creds, self.__channel_options + logging.warning( + "Overriding SSL option from certificate could increase MITM susceptibility!" + ) + self.set_channel_option( + "grpc.ssl_target_name_override", self.__ssl_target_name_override ) - else: - channel = grpc.insecure_channel(self.__target_netloc.netloc) + channel = grpc.secure_channel( + self.__target_netloc.netloc, channel_creds, self.__channel_options + ) if self.__client_class is None: self.set_os() client = self.__client_class(channel) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 2dc5d9f..d3caf08 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -24,16 +24,11 @@ """Python gNMI wrapper to ease usage of gNMI.""" import logging -from collections import OrderedDict from xml.etree.ElementPath import xpath_tokenizer_re -import re -import json -import os from six import string_types from . import proto from . import util -from .xpath_util import get_payload, parse_xpath_to_gnmi_path class Client(object): @@ -256,74 +251,220 @@ def validate_request(request): ) return response_stream - def check_configs(self, configs): - if isinstance(configs, string_types): - logger.debug("Handling as JSON string.") - try: - configs = json.loads(configs) - except: - raise Exception("{0}\n is invalid JSON!".format(configs)) - configs = [configs] - elif isinstance(configs, dict): - logger.debug("Handling already serialized JSON object.") - configs = [configs] - elif not isinstance(configs, (list, set)): - raise Exception("{0} must be an iterable of configs!".format(str(configs))) - return configs - - def create_updates(self, configs, origin, json_ietf=False): - """Check configs, and construct "Update" messages. + def subscribe_xpaths( + self, + xpath_subscriptions, + request_mode="STREAM", + sub_mode="SAMPLE", + encoding="JSON", + sample_interval=_NS_IN_S * 10, + suppress_redundant=False, + heartbeat_interval=None, + ): + """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest + with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, + dictionaries with Subscription attributes for more granularity, or already built Subscription + objects and builds the SubscriptionList. Fields not supplied will be defaulted with the default arguments + to the method. + + Generates a single SubscribeRequest. Parameters ---------- - configs: dict of : - origin: str [DME, device, openconfig] - json_ietf: bool encoding type for Update val (default False) + xpath_subscriptions : str or iterable of str, dict, Subscription + An iterable which is parsed to form the Subscriptions in the SubscriptionList to be passed + to SubscriptionRequest. Strings are parsed as XPaths and defaulted with the default arguments, + dictionaries are treated as dicts of args to pass to the Subscribe init, and Subscription is + treated as simply a pre-made Subscription. + request_mode : proto.gnmi_pb2.SubscriptionList.Mode, optional + Indicates whether STREAM to stream from target, + ONCE to stream once (like a get), + POLL to respond to POLL. + [STREAM, ONCE, POLL] + sub_mode : proto.gnmi_pb2.SubscriptionMode, optional + The default SubscriptionMode on a per Subscription basis in the SubscriptionList. + TARGET_DEFINED indicates that the target (like device/destination) should stream + information however it knows best. This instructs the target to decide between ON_CHANGE + or SAMPLE - e.g. the device gNMI server may understand that we only need RIB updates + as an ON_CHANGE basis as opposed to SAMPLE, and we don't have to explicitly state our + desired behavior. + ON_CHANGE only streams updates when changes occur. + SAMPLE will stream the subscription at a regular cadence/interval. + [TARGET_DEFINED, ON_CHANGE, SAMPLE] + encoding : proto.gnmi_pb2.Encoding, optional + A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data + [JSON, BYTES, PROTO, ASCII, JSON_IETF] + sample_interval : int, optional + Default nanoseconds for SAMPLE to occur. + Defaults to 10 seconds. + suppress_redundant : bool, optional + Indicates whether values that have not changed should be sent in a SAMPLE subscription. + heartbeat_interval : int, optional + Specifies the maximum allowable silent period in nanoseconds when + suppress_redundant is in use. The target should send a value at least once + in the period specified. Also applies in ON_CHANGE. Returns ------- - List of Update messages with val populated. - - If a set of configs contain a common Xpath, the Update must contain - a consolidation of xpath/values for 2 reasons: - - 1. Devices may have a restriction on how many Update messages it will - accept at once. - 2. Some xpath/values are required to be set in same Update because of - dependencies like leafrefs, mandatory settings, and if/when/musts. + subscribe() """ - if not configs: - return [] - configs = self.check_configs(configs) - - xpaths = [] - updates = [] - for config in configs: - xpath = next(iter(config.keys())) - xpaths.append(xpath) - common_xpath = os.path.commonprefix(xpaths) - - if common_xpath: - update_configs = get_payload(configs) - for update_cfg in update_configs: - xpath, payload = update_cfg - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(parse_xpath_to_gnmi_path(xpath, origin=origin)) - if json_ietf: - update.val.json_ietf_val = payload + subscription_list = proto.gnmi_pb2.SubscriptionList() + subscription_list.mode = util.validate_proto_enum( + "mode", + request_mode, + "SubscriptionList.Mode", + proto.gnmi_pb2.SubscriptionList.Mode, + ) + subscription_list.encoding = util.validate_proto_enum( + "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding + ) + if isinstance( + xpath_subscriptions, (string_types, dict, proto.gnmi_pb2.Subscription) + ): + xpath_subscriptions = [xpath_subscriptions] + subscriptions = [] + for xpath_subscription in xpath_subscriptions: + subscription = None + if isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): + subscription = xpath_subscription + elif isinstance(xpath_subscription, string_types): + subscription = proto.gnmi_pb2.Subscription() + subscription.path.CopyFrom( + self.parse_xpath_to_gnmi_path(xpath_subscription) + ) + subscription.mode = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + ) + if sub_mode == "SAMPLE": + subscription.sample_interval = sample_interval + elif isinstance(xpath_subscription, dict): + subscription_dict = {} + if "path" not in xpath_subscription.keys(): + raise Exception("path must be specified in dict!") + if isinstance(xpath_subscription["path"], proto.gnmi_pb2.Path): + subscription_dict["path"] = xpath_subscription["path"] + elif isinstance(xpath_subscription["path"], string_types): + subscription_dict["path"] = self.parse_xpath_to_gnmi_path( + xpath_subscription["path"] + ) else: - update.val.json_val = payload - updates.append(update) - return updates - else: - for config in configs: - top_element = next(iter(config.keys())) - update = proto.gnmi_pb2.Update() - update.path.CopyFrom(parse_xpath_to_gnmi_path(top_element)) - config = config.pop(top_element) - if json_ietf: - update.val.json_ietf_val = json.dumps(config).encode("utf-8") + raise Exception("path must be string or Path proto!") + sub_mode_name = ( + sub_mode + if "mode" not in xpath_subscription.keys() + else xpath_subscription["mode"] + ) + subscription_dict["mode"] = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + ) + if sub_mode_name == "SAMPLE": + subscription_dict["sample_interval"] = ( + sample_interval + if "sample_interval" not in xpath_subscription.keys() + else xpath_subscription["sample_interval"] + ) + if "suppress_redundant" in xpath_subscription.keys(): + subscription_dict["suppress_redundant"] = xpath_subscription[ + "suppress_redundant" + ] + if sub_mode_name != "TARGET_DEFINED": + if "heartbeat_interval" in xpath_subscription.keys(): + subscription_dict["heartbeat_interval"] = xpath_subscription[ + "heartbeat_interval" + ] + subscription = proto.gnmi_pb2.Subscription(**subscription_dict) + else: + raise Exception("path must be string, dict, or Subscription proto!") + subscriptions.append(subscription) + subscription_list.subscription.extend(subscriptions) + return self.subscribe([subscription_list]) + + def parse_xpath_to_gnmi_path(self, xpath, origin=None): + """Parses an XPath to proto.gnmi_pb2.Path. + This function should be overridden by any child classes for origin logic. + + Effectively wraps the std XML XPath tokenizer and traverses + the identified groups. Parsing robustness needs to be validated. + Probably best to formalize as a state machine sometime. + TODO: Formalize tokenizer traversal via state machine. + """ + if not isinstance(xpath, string_types): + raise Exception("xpath must be a string!") + path = proto.gnmi_pb2.Path() + if origin: + if not isinstance(origin, string_types): + raise Exception("origin must be a string!") + path.origin = origin + curr_elem = proto.gnmi_pb2.PathElem() + in_filter = False + just_filtered = False + curr_key = None + # TODO: Lazy + xpath = xpath.strip("/") + xpath_elements = xpath_tokenizer_re.findall(xpath) + path_elems = [] + for index, element in enumerate(xpath_elements): + # stripped initial /, so this indicates a completed element + if element[0] == "/": + if not curr_elem.name: + raise Exception( + "Current PathElem has no name yet is trying to be pushed to path! Invalid XPath?" + ) + path_elems.append(curr_elem) + curr_elem = proto.gnmi_pb2.PathElem() + continue + # We are entering a filter + elif element[0] == "[": + in_filter = True + continue + # We are exiting a filter + elif element[0] == "]": + in_filter = False + continue + # If we're not in a filter then we're a PathElem name + elif not in_filter: + curr_elem.name = element[1] + # Skip blank spaces + elif not any([element[0], element[1]]): + continue + # If we're in the filter and just completed a filter expr, + # "and" as a junction should just be ignored. + elif in_filter and just_filtered and element[1] == "and": + just_filtered = False + continue + # Otherwise we're in a filter and this term is a key name + elif curr_key is None: + curr_key = element[1] + continue + # Otherwise we're an operator or the key value + elif curr_key is not None: + # I think = is the only possible thing to support with PathElem syntax as is + if element[0] in [">", "<"]: + raise Exception("Only = supported as filter operand!") + if element[0] == "=": + continue else: - update.val.json_val = json.dumps(config).encode("utf-8") - updates.append(update) - return updates + # We have a full key here, put it in the map + if curr_key in curr_elem.key.keys(): + raise Exception("Key already in key map!") + curr_elem.key[curr_key] = element[0].strip("'\"") + curr_key = None + just_filtered = True + # Keys/filters in general should be totally cleaned up at this point. + if curr_key: + raise Exception("Hanging key filter! Incomplete XPath?") + # If we have a dangling element that hasn't been completed due to no + # / element then let's just append the final element. + if curr_elem: + path_elems.append(curr_elem) + curr_elem = None + if any([curr_elem, curr_key, in_filter]): + raise Exception("Unfinished elements in XPath parsing!") + path.elem.extend(path_elems) + return path diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 78e036e..4aab05b 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -25,15 +25,9 @@ import logging -import json -import os from six import string_types -from . import proto, util -from .client import Client -from .xpath_util import parse_xpath_to_gnmi_path - -logger = logging.getLogger(__name__) +from .client import Client, proto, util class NXClient(Client): @@ -59,117 +53,11 @@ class NXClient(Client): >>> print(capabilities) """ - def delete_xpaths(self, xpaths, prefix=None): - """A convenience wrapper for set() which constructs Paths from supplied xpaths - to be passed to set() as the delete parameter. - - Parameters - ---------- - xpaths : iterable of str - XPaths to specify to be deleted. - If prefix is specified these strings are assumed to be the suffixes. - prefix : str - The XPath prefix to apply to all XPaths for deletion. - - Returns - ------- - set() - """ - if isinstance(xpaths, string_types): - xpaths = [xpaths] - paths = [] - for xpath in xpaths: - if prefix: - if prefix.endswith("/") and xpath.startswith("/"): - xpath = "{prefix}{xpath}".format( - prefix=prefix[:-1], xpath=xpath[1:] - ) - elif prefix.endswith("/") or xpath.startswith("/"): - xpath = "{prefix}{xpath}".format(prefix=prefix, xpath=xpath) - else: - xpath = "{prefix}/{xpath}".format(prefix=prefix, xpath=xpath) - paths.append(parse_xpath_to_gnmi_path(xpath)) - return self.set(deletes=paths) - - def set_json( - self, - update_json_configs=None, - replace_json_configs=None, - origin="device", - json_ietf=False, - ): - """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. - All parameters are optional, but at least one must be present. - - This method expects JSON in the same format as what you might send via the native gRPC interface - with a fully modeled configuration which is then parsed to meet the gNMI implementation. - - Parameters - ---------- - update_json_configs : iterable of JSON configurations, optional - JSON configs to apply as updates. - replace_json_configs : iterable of JSON configurations, optional - JSON configs to apply as replacements. - origin : openconfig, device, or DME - - Returns - ------- - set() - """ - if not any([update_json_configs, replace_json_configs]): - raise Exception("Must supply at least one set of configurations to method!") - - updates = self.create_updates( - update_json_configs, origin=origin, json_ietf=json_ietf - ) - replaces = self.create_updates( - replace_json_configs, origin=origin, json_ietf=json_ietf - ) - for update in updates + replaces: - logger.debug("\nGNMI set:\n{0}\n{1}".format(9 * "=", str(update))) - - return self.set(updates=updates, replaces=replaces) - - def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON", origin="openconfig"): - """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. - - Parameters - ---------- - xpaths : iterable of str or str - An iterable of XPath strings to request data of - If simply a str, wraps as a list for convenience - data_type : proto.gnmi_pb2.GetRequest.DataType, optional - A direct value or key from the GetRequest.DataType enum - [ALL, CONFIG, STATE, OPERATIONAL] - encoding : proto.gnmi_pb2.GetRequest.Encoding, optional - A direct value or key from the Encoding enum - [JSON, JSON_IETF] + def get(self, *args, **kwargs): + raise NotImplementedError("Get not yet supported on NX-OS!") - Returns - ------- - get() - """ - supported_encodings = ["JSON"] - encoding = util.validate_proto_enum( - "encoding", - encoding, - "Encoding", - proto.gnmi_pb2.Encoding, - supported_encodings, - ) - gnmi_path = None - if isinstance(xpaths, (list, set)): - gnmi_path = [] - for xpath in set(xpaths): - gnmi_path.append(parse_xpath_to_gnmi_path(xpath, origin)) - elif isinstance(xpaths, string_types): - gnmi_path = [parse_xpath_to_gnmi_path(xpaths, origin)] - else: - raise Exception( - "xpaths must be a single xpath string or iterable of xpath strings!" - ) - logger.debug("GNMI get:\n{0}\n{1}".format(9 * "=", str(gnmi_path))) - return self.get(gnmi_path, data_type=data_type, encoding=encoding) + def set(self, *args, **kwargs): + raise NotImplementedError("Set not yet supported on NX-OS!") def subscribe_xpaths( self, @@ -178,7 +66,8 @@ def subscribe_xpaths( sub_mode="SAMPLE", encoding="PROTO", sample_interval=Client._NS_IN_S * 10, - origin="openconfig", + suppress_redundant=False, + heartbeat_interval=None, ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -211,71 +100,69 @@ def subscribe_xpaths( sample_interval : int, optional Default nanoseconds for sample to occur. Defaults to 10 seconds. + suppress_redundant : bool, optional + Indicates whether values that have not changed should be sent in a SAMPLE subscription. + heartbeat_interval : int, optional + Specifies the maximum allowable silent period in nanoseconds when + suppress_redundant is in use. The target should send a value at least once + in the period specified. Returns ------- subscribe() """ supported_request_modes = ["STREAM", "ONCE", "POLL"] - supported_encodings = ["JSON", "PROTO"] - supported_sub_modes = ["ON_CHANGE", "SAMPLE"] - subscription_list = proto.gnmi_pb2.SubscriptionList() - subscription_list.mode = util.validate_proto_enum( + request_mode = util.validate_proto_enum( "mode", request_mode, "SubscriptionList.Mode", proto.gnmi_pb2.SubscriptionList.Mode, - supported_request_modes, + subset=supported_request_modes, + return_name=True, ) - subscription_list.encoding = util.validate_proto_enum( + supported_encodings = ["JSON", "PROTO"] + encoding = util.validate_proto_enum( "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding, - supported_encodings, + subset=supported_encodings, + return_name=True, ) - if isinstance(xpath_subscriptions, string_types): - xpath_subscriptions = [xpath_subscriptions] - subscriptions = [] - for xpath_subscription in xpath_subscriptions: - subscription = None - if isinstance(xpath_subscription, string_types): - subscription = proto.gnmi_pb2.Subscription() - subscription.path.CopyFrom( - parse_xpath_to_gnmi_path(xpath_subscription, origin) - ) - subscription.mode = util.validate_proto_enum( - "sub_mode", - sub_mode, - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription.sample_interval = sample_interval - elif isinstance(xpath_subscription, dict): - path = parse_xpath_to_gnmi_path(xpath_subscription["path"], origin) - arg_dict = { - "path": path, - "mode": sub_mode, - "sample_interval": sample_interval, - } - arg_dict.update(xpath_subscription) - if "mode" in arg_dict: - arg_dict["mode"] = util.validate_proto_enum( - "sub_mode", - arg_dict["mode"], - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription = proto.gnmi_pb2.Subscription(**arg_dict) - elif isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): - subscription = xpath_subscription - else: - raise Exception("xpath in list must be xpath or dict/Path!") - subscriptions.append(subscription) - subscription_list.subscription.extend(subscriptions) - logger.debug( - "GNMI subscribe:\n{0}\n{1}".format(15 * "=", str(subscription_list)) + supported_sub_modes = ["ON_CHANGE", "SAMPLE"] + sub_mode = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + subset=supported_sub_modes, + return_name=True, + ) + return super(NXClient, self).subscribe_xpaths( + xpath_subscriptions, + request_mode, + sub_mode, + encoding, + sample_interval, + suppress_redundant, + heartbeat_interval, ) - return self.subscribe([subscription_list]) + + def parse_xpath_to_gnmi_path(self, xpath, origin=None): + """Attempts to determine whether origin should be YANG (device) or DME. + Errors on OpenConfig until support is present. + """ + if xpath.startswith("openconfig"): + raise NotImplementedError( + "OpenConfig data models not yet supported on NX-OS!" + ) + if origin is None: + if any( + map(xpath.startswith, ["Cisco-NX-OS-device", "/Cisco-NX-OS-device"]) + ): + origin = "device" + # Remove the module + xpath = xpath.split(":", 1)[1] + else: + origin = "DME" + return super(NXClient, self).parse_xpath_to_gnmi_path(xpath, origin) diff --git a/src/cisco_gnmi/util.py b/src/cisco_gnmi/util.py index f65f104..7d205a6 100644 --- a/src/cisco_gnmi/util.py +++ b/src/cisco_gnmi/util.py @@ -60,7 +60,9 @@ def gen_target_netloc(target, netloc_prefix="//", default_port=9339): return target_netloc -def validate_proto_enum(value_name, value, enum_name, enum, subset=None): +def validate_proto_enum( + value_name, value, enum_name, enum, subset=None, return_name=False +): """Helper function to validate an enum against the proto enum wrapper.""" enum_value = None if value not in enum.keys() and value not in enum.values(): @@ -91,11 +93,15 @@ def validate_proto_enum(value_name, value, enum_name, enum, subset=None): ) if enum_value not in resolved_subset: raise Exception( - "{name}={value} not in subset {subset}!".format( - name=value_name, value=enum_value, subset=resolved_subset + "{name}={value} ({actual_value}) not in subset {subset} ({actual_subset})!".format( + name=value_name, + value=value, + actual_value=enum_value, + subset=subset, + actual_subset=resolved_subset, ) ) - return enum_value + return enum_value if not return_name else enum.Name(enum_value) def get_cert_from_target(target_netloc): diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 0fa5e20..6c9e525 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -25,15 +25,9 @@ import json import logging -import os from six import string_types -from . import proto, util -from .client import Client -from .xpath_util import parse_xpath_to_gnmi_path - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +from .client import Client, proto, util class XEClient(Client): @@ -111,16 +105,10 @@ def delete_xpaths(self, xpaths, prefix=None): xpath = "{prefix}{xpath}".format(prefix=prefix, xpath=xpath) else: xpath = "{prefix}/{xpath}".format(prefix=prefix, xpath=xpath) - paths.append(parse_xpath_to_gnmi_path(xpath)) + paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json( - self, - update_json_configs=None, - replace_json_configs=None, - origin="device", - json_ietf=True, - ): + def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -133,7 +121,8 @@ def set_json( JSON configs to apply as updates. replace_json_configs : iterable of JSON configurations, optional JSON configs to apply as replacements. - origin : openconfig, device, or DME + ietf : bool, optional + Use JSON_IETF vs JSON. Returns ------- @@ -142,18 +131,49 @@ def set_json( if not any([update_json_configs, replace_json_configs]): raise Exception("Must supply at least one set of configurations to method!") - updates = self.create_updates( - update_json_configs, origin=origin, json_ietf=json_ietf - ) - replaces = self.create_updates( - replace_json_configs, origin=origin, json_ietf=json_ietf - ) - for update in updates + replaces: - logger.debug("\nGNMI set:\n{0}\n{1}".format(9 * "=", str(update))) + def check_configs(name, configs): + if isinstance(name, string_types): + logging.debug("Handling %s as JSON string.", name) + try: + configs = json.loads(configs) + except: + raise Exception("{name} is invalid JSON!".format(name=name)) + configs = [configs] + elif isinstance(name, dict): + logging.debug("Handling %s as already serialized JSON object.", name) + configs = [configs] + elif not isinstance(configs, (list, set)): + raise Exception( + "{name} must be an iterable of configs!".format(name=name) + ) + return configs + def create_updates(name, configs): + if not configs: + return None + configs = check_configs(name, configs) + updates = [] + for config in configs: + if not isinstance(config, dict): + raise Exception("config must be a JSON object!") + if len(config.keys()) > 1: + raise Exception("config should only target one YANG module!") + top_element = next(iter(config.keys())) + update = proto.gnmi_pb2.Update() + update.path.CopyFrom(self.parse_xpath_to_gnmi_path(top_element)) + config = config.pop(top_element) + if ietf: + update.val.json_ietf_val = json.dumps(config).encode("utf-8") + else: + update.val.json_val = json.dumps(config).encode("utf-8") + updates.append(update) + return updates + + updates = create_updates("update_json_configs", update_json_configs) + replaces = create_updates("replace_json_configs", replace_json_configs) return self.set(updates=updates, replaces=replaces) - def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None): + def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF"): """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. Parameters @@ -182,16 +202,13 @@ def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF", origin=None) ) gnmi_path = None if isinstance(xpaths, (list, set)): - gnmi_path = [] - for xpath in set(xpaths): - gnmi_path.append(parse_xpath_to_gnmi_path(xpath, origin)) + gnmi_path = map(self.parse_xpath_to_gnmi_path, set(xpaths)) elif isinstance(xpaths, string_types): - gnmi_path = [parse_xpath_to_gnmi_path(xpaths, origin)] + gnmi_path = [self.parse_xpath_to_gnmi_path(xpaths)] else: raise Exception( "xpaths must be a single xpath string or iterable of xpath strings!" ) - logger.debug("GNMI get:\n{0}\n{1}".format(9 * "=", str(gnmi_path))) return self.get(gnmi_path, data_type=data_type, encoding=encoding) def subscribe_xpaths( @@ -201,7 +218,8 @@ def subscribe_xpaths( sub_mode="SAMPLE", encoding="JSON_IETF", sample_interval=Client._NS_IN_S * 10, - origin="openconfig", + suppress_redundant=False, + heartbeat_interval=None, ): """A convenience wrapper of subscribe() which aids in building of SubscriptionRequest with request as subscribe SubscriptionList. This method accepts an iterable of simply xpath strings, @@ -219,86 +237,75 @@ def subscribe_xpaths( dictionaries are treated as dicts of args to pass to the Subscribe init, and Subscription is treated as simply a pre-made Subscription. request_mode : proto.gnmi_pb2.SubscriptionList.Mode, optional - Indicates whether STREAM to stream from target, - ONCE to stream once (like a get), - POLL to respond to POLL. - [STREAM, ONCE, POLL] + Indicates whether STREAM to stream from target. + [STREAM] sub_mode : proto.gnmi_pb2.SubscriptionMode, optional The default SubscriptionMode on a per Subscription basis in the SubscriptionList. - ON_CHANGE only streams updates when changes occur. SAMPLE will stream the subscription at a regular cadence/interval. - [ON_CHANGE, SAMPLE] + [SAMPLE] encoding : proto.gnmi_pb2.Encoding, optional A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data - [JSON, JSON_IETF] + [JSON_IETF] sample_interval : int, optional Default nanoseconds for sample to occur. Defaults to 10 seconds. + suppress_redundant : bool, optional + Indicates whether values that have not changed should be sent in a SAMPLE subscription. + heartbeat_interval : int, optional + Specifies the maximum allowable silent period in nanoseconds when + suppress_redundant is in use. The target should send a value at least once + in the period specified. Returns ------- subscribe() """ - supported_request_modes = ["STREAM", "ONCE", "POLL"] - supported_encodings = ["JSON", "JSON_IETF"] - supported_sub_modes = ["ON_CHANGE", "SAMPLE"] - subscription_list = proto.gnmi_pb2.SubscriptionList() - subscription_list.mode = util.validate_proto_enum( + supported_request_modes = ["STREAM"] + request_mode = util.validate_proto_enum( "mode", request_mode, "SubscriptionList.Mode", proto.gnmi_pb2.SubscriptionList.Mode, - supported_request_modes, + subset=supported_request_modes, + return_name=True, ) - subscription_list.encoding = util.validate_proto_enum( + supported_encodings = ["JSON_IETF"] + encoding = util.validate_proto_enum( "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding, - supported_encodings, + subset=supported_encodings, + return_name=True, ) - if isinstance(xpath_subscriptions, string_types): - xpath_subscriptions = [xpath_subscriptions] - subscriptions = [] - for xpath_subscription in xpath_subscriptions: - subscription = None - if isinstance(xpath_subscription, string_types): - subscription = proto.gnmi_pb2.Subscription() - subscription.path.CopyFrom( - parse_xpath_to_gnmi_path(xpath_subscription, origin) - ) - subscription.mode = util.validate_proto_enum( - "sub_mode", - sub_mode, - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription.sample_interval = sample_interval - elif isinstance(xpath_subscription, dict): - path = parse_xpath_to_gnmi_path(xpath_subscription["path"], origin) - arg_dict = { - "path": path, - "mode": sub_mode, - "sample_interval": sample_interval, - } - arg_dict.update(xpath_subscription) - if "mode" in arg_dict: - arg_dict["mode"] = util.validate_proto_enum( - "sub_mode", - arg_dict["mode"], - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - supported_sub_modes, - ) - subscription = proto.gnmi_pb2.Subscription(**arg_dict) - elif isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): - subscription = xpath_subscription - else: - raise Exception("xpath in list must be xpath or dict/Path!") - subscriptions.append(subscription) - subscription_list.subscription.extend(subscriptions) - logger.debug( - "GNMI subscribe:\n{0}\n{1}".format(15 * "=", str(subscription_list)) + supported_sub_modes = ["SAMPLE"] + sub_mode = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + subset=supported_sub_modes, + return_name=True, ) - return self.subscribe([subscription_list]) + return super(XEClient, self).subscribe_xpaths( + xpath_subscriptions, + request_mode, + sub_mode, + encoding, + sample_interval, + suppress_redundant, + heartbeat_interval, + ) + + def parse_xpath_to_gnmi_path(self, xpath, origin=None): + """Naively tries to intelligently (non-sequitur!) origin + Otherwise assume rfc7951 + legacy is not considered + """ + if origin is None: + # naive but effective + if ":" not in xpath: + origin = "openconfig" + else: + origin = "rfc7951" + return super(XEClient, self).parse_xpath_to_gnmi_path(xpath, origin) diff --git a/src/cisco_gnmi/xpath_util.py b/src/cisco_gnmi/xpath_util.py deleted file mode 100644 index 698a901..0000000 --- a/src/cisco_gnmi/xpath_util.py +++ /dev/null @@ -1,499 +0,0 @@ -import os -import re -import json -import logging -from xml.etree.ElementPath import xpath_tokenizer_re -from six import string_types -from . import proto - - -log = logging.getLogger(__name__) - - -def parse_xpath_to_gnmi_path(xpath, origin=None): - """Parses an XPath to proto.gnmi_pb2.Path. - This function should be overridden by any child classes for origin logic. - - Effectively wraps the std XML XPath tokenizer and traverses - the identified groups. Parsing robustness needs to be validated. - Probably best to formalize as a state machine sometime. - TODO: Formalize tokenizer traversal via state machine. - """ - if not isinstance(xpath, string_types): - raise Exception("xpath must be a string!") - path = proto.gnmi_pb2.Path() - if origin: - if not isinstance(origin, string_types): - raise Exception("origin must be a string!") - path.origin = origin - curr_elem = proto.gnmi_pb2.PathElem() - in_filter = False - just_filtered = False - curr_key = None - # TODO: Lazy - xpath = xpath.strip("/") - xpath_elements = xpath_tokenizer_re.findall(xpath) - path_elems = [] - for index, element in enumerate(xpath_elements): - # stripped initial /, so this indicates a completed element - if element[0] == "/": - if not curr_elem.name: - raise Exception( - "Current PathElem has no name yet is trying to be pushed to path! Invalid XPath?" - ) - path_elems.append(curr_elem) - curr_elem = proto.gnmi_pb2.PathElem() - continue - # We are entering a filter - elif element[0] == "[": - in_filter = True - continue - # We are exiting a filter - elif element[0] == "]": - in_filter = False - continue - # If we're not in a filter then we're a PathElem name - elif not in_filter: - curr_elem.name = element[1] - # Skip blank spaces - elif not any([element[0], element[1]]): - continue - # If we're in the filter and just completed a filter expr, - # "and" as a junction should just be ignored. - elif in_filter and just_filtered and element[1] == "and": - just_filtered = False - continue - # Otherwise we're in a filter and this term is a key name - elif curr_key is None: - curr_key = element[1] - continue - # Otherwise we're an operator or the key value - elif curr_key is not None: - # I think = is the only possible thing to support with PathElem syntax as is - if element[0] in [">", "<"]: - raise Exception("Only = supported as filter operand!") - if element[0] == "=": - continue - else: - # We have a full key here, put it in the map - if curr_key in curr_elem.key.keys(): - raise Exception("Key already in key map!") - curr_elem.key[curr_key] = element[0].strip("'\"") - curr_key = None - just_filtered = True - # Keys/filters in general should be totally cleaned up at this point. - if curr_key: - raise Exception("Hanging key filter! Incomplete XPath?") - # If we have a dangling element that hasn't been completed due to no - # / element then let's just append the final element. - if curr_elem: - path_elems.append(curr_elem) - curr_elem = None - if any([curr_elem, curr_key, in_filter]): - raise Exception("Unfinished elements in XPath parsing!") - path.elem.extend(path_elems) - return path - - -def combine_configs(payload, last_xpath, cfg): - """Walking from end to finish, 2 xpaths merge, so combine them. - |--config - |---last xpath config--| - ----| |--config - | - | pick these up --> |--config - |---this xpath config--| - |--config - Parameters - ---------- - payload: dict of partial payload - last_xpath: last xpath that was processed - xpath: colliding xpath - config: dict of values associated to colliding xpath - """ - xpath, config, is_key = cfg - lp = last_xpath.split("/") - xp = xpath.split("/") - base = [] - top = "" - for i, seg in enumerate(zip(lp, xp)): - if seg[0] != seg[1]: - top = seg[1] - break - base = "/" + "/".join(xp[i:]) - cfg = (base, config, False) - extended_payload = {top: xpath_to_json([cfg])} - payload.update(extended_payload) - return payload - - -def xpath_to_json(configs, last_xpath="", payload={}): - """Try to combine Xpaths/values into a common payload (recursive). - - Parameters - ---------- - configs: tuple of xpath/value dict - last_xpath: str of last xpath that was recusivly processed. - payload: dict being recursively built for JSON transformation. - - Returns - ------- - dict of combined xpath/value dict. - """ - for i, cfg in enumerate(configs, 1): - xpath, config, is_key = cfg - if last_xpath and xpath not in last_xpath: - # Branched config here |---config - # |---last xpath config--| - # --| |---config - # |---this xpath config - payload = combine_configs(payload, last_xpath, cfg) - return xpath_to_json(configs[i:], xpath, payload) - xpath_segs = xpath.split("/") - xpath_segs.reverse() - for seg in xpath_segs: - if not seg: - continue - if payload: - if is_key: - if seg in payload: - if isinstance(payload[seg], list): - payload[seg].append(config) - elif isinstance(payload[seg], dict): - payload[seg].update(config) - else: - payload.update(config) - payload = {seg: [payload]} - else: - config.update(payload) - payload = {seg: config} - return xpath_to_json(configs[i:], xpath, payload) - else: - if is_key: - payload = {seg: [config]} - else: - payload = {seg: config} - return xpath_to_json(configs[i:], xpath, payload) - return payload - - -# Pattern to detect keys in an xpath -RE_FIND_KEYS = re.compile(r"\[.*?\]") - - -def get_payload(configs): - """Common Xpaths were detected so try to consolidate them. - - Parameter - --------- - configs: list of {xpath: {name: value}} dicts - """ - # Number of updates are limited so try to consolidate into lists. - xpaths_cfg = [] - first_key = set() - # Find first common keys for all xpaths_cfg of collection. - for config in configs: - xpath = next(iter(config.keys())) - - # Change configs to tuples (xpath, config) for easier management - xpaths_cfg.append((xpath, config[xpath])) - - xpath_split = xpath.split("/") - for seg in xpath_split: - if "[" in seg: - first_key.add(seg) - break - - # Common first key/configs represents one GNMI update - updates = [] - for key in first_key: - update = [] - remove_cfg = [] - for config in xpaths_cfg: - xpath, cfg = config - if key in xpath: - update.append(config) - else: - for k, v in cfg.items(): - if '[{0}="{1}"]'.format(k, v) not in key: - break - else: - # This cfg sets the first key so we don't need it - remove_cfg.append((xpath, cfg)) - if update: - for upd in update: - # Remove this config out of main list - xpaths_cfg.remove(upd) - for rem_cfg in remove_cfg: - # Sets a key in update path so remove it - xpaths_cfg.remove(rem_cfg) - updates.append(update) - break - - # Add remaining configs to updates - if xpaths_cfg: - updates.append(xpaths_cfg) - - # Combine all xpath configs of each update if possible - xpaths = [] - compressed_updates = [] - for update in updates: - xpath_consolidated = {} - config_compressed = [] - for seg in update: - xpath, config = seg - if xpath in xpath_consolidated: - xpath_consolidated[xpath].update(config) - else: - xpath_consolidated[xpath] = config - config_compressed.append((xpath, xpath_consolidated[xpath])) - xpaths.append(xpath) - - # Now get the update path for this batch of configs - common_xpath = os.path.commonprefix(xpaths) - cfg_compressed = [] - keys = [] - - # Need to reverse the configs to build the dict correctly - config_compressed.reverse() - for seg in config_compressed: - is_key = False - prepend_path = "" - xpath, config = seg - end_path = xpath[len(common_xpath) :] - if end_path.startswith("["): - # Don't start payload with a list - tmp = common_xpath.split("/") - prepend_path = "/" + tmp.pop() - common_xpath = "/".join(tmp) - end_path = prepend_path + end_path - - # Building json, need to identify configs that set keys - for key in keys: - if [k for k in config.keys() if k in key]: - is_key = True - keys += re.findall(RE_FIND_KEYS, end_path) - cfg_compressed.append((end_path, config, is_key)) - - update = (common_xpath, cfg_compressed) - compressed_updates.append(update) - - updates = [] - for update in compressed_updates: - common_xpath, cfgs = update - payload = xpath_to_json(cfgs) - updates.append((common_xpath, json.dumps(payload).encode("utf-8"))) - return updates - - -def xml_path_to_path_elem(request): - """Convert XML Path Language 1.0 Xpath to gNMI Path/PathElement. - - Modeled after YANG/NETCONF Xpaths. - - References: - * https://www.w3.org/TR/1999/REC-xpath-19991116/#location-paths - * https://www.w3.org/TR/1999/REC-xpath-19991116/#path-abbrev - * https://tools.ietf.org/html/rfc6020#section-6.4 - * https://tools.ietf.org/html/rfc6020#section-9.13 - * https://tools.ietf.org/html/rfc6241 - - Parameters - --------- - request: dict containing request namespace and nodes to be worked on. - namespace: dict of : - nodes: list of dict - : Xpath pointing to resource - : value to set resource to - : equivelant NETCONF edit-config operation - - Returns - ------- - tuple: namespace_modules, message dict, origin - namespace_modules: dict of : - Needed for future support. - message dict: 4 lists containing possible updates, replaces, - deletes, or gets derived form input nodes. - origin str: DME, device, or openconfig - """ - - paths = [] - message = { - "update": [], - "replace": [], - "delete": [], - "get": [], - } - if "nodes" not in request: - # TODO: raw rpc? - return paths - else: - namespace_modules = {} - origin = "DME" - for prefix, nspace in request.get("namespace", {}).items(): - if "/Cisco-IOS-" in nspace: - module = nspace[nspace.rfind("/") + 1 :] - elif "/cisco-nx" in nspace: # NXOS lowercases namespace - module = "Cisco-NX-OS-device" - elif "/openconfig.net" in nspace: - module = "openconfig-" - module += nspace[nspace.rfind("/") + 1 :] - elif "urn:ietf:params:xml:ns:yang:" in nspace: - module = nspace.replace("urn:ietf:params:xml:ns:yang:", "") - if module: - namespace_modules[prefix] = module - - for node in request.get("nodes", []): - if "xpath" not in node: - log.error("Xpath is not in message") - else: - xpath = node["xpath"] - value = node.get("value", "") - edit_op = node.get("edit-op", "") - - for pfx, ns in namespace_modules.items(): - # NXOS does not support prefixes yet so clear them out - if pfx in xpath and "openconfig" in ns: - origin = "openconfig" - xpath = xpath.replace(pfx + ":", "") - if isinstance(value, string_types): - value = value.replace(pfx + ":", "") - elif pfx in xpath and "device" in ns: - origin = "device" - xpath = xpath.replace(pfx + ":", "") - if isinstance(value, string_types): - value = value.replace(pfx + ":", "") - if edit_op: - if edit_op in ["create", "merge", "replace"]: - xpath_lst = xpath.split("/") - name = xpath_lst.pop() - xpath = "/".join(xpath_lst) - if edit_op == "replace": - if not message["replace"]: - message["replace"] = [{xpath: {name: value}}] - else: - message["replace"].append({xpath: {name: value}}) - else: - if not message["update"]: - message["update"] = [{xpath: {name: value}}] - else: - message["update"].append({xpath: {name: value}}) - elif edit_op in ["delete", "remove"]: - if message["delete"]: - message["delete"].add(xpath) - else: - message["delete"] = set(xpath) - else: - message["get"].append(xpath) - return namespace_modules, message, origin - - -if __name__ == "__main__": - from pprint import pprint as pp - import grpc - from cisco_gnmi.auth import CiscoAuthPlugin - from cisco_gnmi.client import Client - - channel = grpc.secure_channel( - "127.0.0.1:9339", - grpc.composite_channel_credentials( - grpc.ssl_channel_credentials(), - grpc.metadata_call_credentials(CiscoAuthPlugin("admin", "its_a_secret")), - ), - ) - client = Client(channel) - request = { - "namespace": {"oc-acl": "http://openconfig.net/yang/acl"}, - "nodes": [ - { - "value": "testacl", - "xpath": "/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name", - "edit-op": "merge", - }, - { - "value": "ACL_IPV4", - "xpath": "/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type", - "edit-op": "merge", - }, - { - "value": "10", - "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', - "edit-op": "merge", - }, - { - "value": "20.20.20.1/32", - "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', - "edit-op": "merge", - }, - { - "value": "IP_TCP", - "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', - "edit-op": "merge", - }, - { - "value": "10.10.10.10/32", - "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', - "edit-op": "merge", - }, - { - "value": "DROP", - "xpath": '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', - "edit-op": "merge", - }, - ], - } - modules, message, origin = xml_path_to_path_elem(request) - pp(modules) - pp(message) - pp(origin) - """ - # Expected output - ================= - {'oc-acl': 'openconfig-acl'} - {'delete': [], - 'get': [], - 'replace': [], - 'update': [{'/acl/acl-sets/acl-set': {'name': 'testacl'}}, - {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}}]} - 'openconfig' - """ - # Feed converted XML Path Language 1.0 Xpaths to create updates - updates = client.create_updates(message["update"], origin) - pp(updates) - """ - # Expected output - ================= - [path { - origin: "openconfig" - elem { - name: "acl" - } - elem { - name: "acl-sets" - } - elem { - name: "acl-set" - key { - key: "name" - value: "testacl" - } - key { - key: "type" - value: "ACL_IPV4" - } - } - elem { - name: "acl-entries" - } - } - val { - json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" - } - ] - # update is now ready to be sent through gNMI SetRequest - """ diff --git a/src/cisco_gnmi/xr.py b/src/cisco_gnmi/xr.py index 627283c..f1bc809 100644 --- a/src/cisco_gnmi/xr.py +++ b/src/cisco_gnmi/xr.py @@ -279,10 +279,10 @@ def subscribe_xpaths( desired behavior. ON_CHANGE only streams updates when changes occur. SAMPLE will stream the subscription at a regular cadence/interval. - [TARGET_DEFINED, ON_CHANGE, SAMPLE] + [ON_CHANGE, SAMPLE] encoding : proto.gnmi_pb2.Encoding, optional A member of the proto.gnmi_pb2.Encoding enum specifying desired encoding of returned data - [JSON, BYTES, PROTO, ASCII, JSON_IETF] + [PROTO] sample_interval : int, optional Default nanoseconds for sample to occur. Defaults to 10 seconds. @@ -297,62 +297,42 @@ def subscribe_xpaths( ------- subscribe() """ - subscription_list = proto.gnmi_pb2.SubscriptionList() - subscription_list.mode = util.validate_proto_enum( + supported_request_modes = ["STREAM", "ONCE", "POLL"] + request_mode = util.validate_proto_enum( "mode", request_mode, "SubscriptionList.Mode", proto.gnmi_pb2.SubscriptionList.Mode, + subset=supported_request_modes, + return_name=True, ) - subscription_list.encoding = util.validate_proto_enum( - "encoding", encoding, "Encoding", proto.gnmi_pb2.Encoding + supported_encodings = ["PROTO"] + encoding = util.validate_proto_enum( + "encoding", + encoding, + "Encoding", + proto.gnmi_pb2.Encoding, + subset=supported_encodings, + return_name=True, + ) + supported_sub_modes = ["ON_CHANGE", "SAMPLE"] + sub_mode = util.validate_proto_enum( + "sub_mode", + sub_mode, + "SubscriptionMode", + proto.gnmi_pb2.SubscriptionMode, + subset=supported_sub_modes, + return_name=True, + ) + return super(XRClient, self).subscribe_xpaths( + xpath_subscriptions, + request_mode, + sub_mode, + encoding, + sample_interval, + suppress_redundant, + heartbeat_interval, ) - if isinstance(xpath_subscriptions, string_types): - xpath_subscriptions = [xpath_subscriptions] - subscriptions = [] - for xpath_subscription in xpath_subscriptions: - subscription = None - if isinstance(xpath_subscription, string_types): - subscription = proto.gnmi_pb2.Subscription() - subscription.path.CopyFrom( - self.parse_xpath_to_gnmi_path(xpath_subscription) - ) - subscription.mode = util.validate_proto_enum( - "sub_mode", - sub_mode, - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - ) - subscription.sample_interval = sample_interval - subscription.suppress_redundant = suppress_redundant - if heartbeat_interval: - subscription.heartbeat_interval = heartbeat_interval - elif isinstance(xpath_subscription, dict): - path = self.parse_xpath_to_gnmi_path(xpath_subscription["path"]) - arg_dict = { - "path": path, - "mode": sub_mode, - "sample_interval": sample_interval, - "suppress_redundant": suppress_redundant, - } - if heartbeat_interval: - arg_dict["heartbeat_interval"] = heartbeat_interval - arg_dict.update(xpath_subscription) - if "mode" in arg_dict: - arg_dict["mode"] = util.validate_proto_enum( - "sub_mode", - arg_dict["mode"], - "SubscriptionMode", - proto.gnmi_pb2.SubscriptionMode, - ) - subscription = proto.gnmi_pb2.Subscription(**arg_dict) - elif isinstance(xpath_subscription, proto.gnmi_pb2.Subscription): - subscription = xpath_subscription - else: - raise Exception("xpath in list must be xpath or dict/Path!") - subscriptions.append(subscription) - subscription_list.subscription.extend(subscriptions) - return self.subscribe([subscription_list]) def parse_xpath_to_gnmi_path(self, xpath, origin=None): """No origin specified implies openconfig @@ -366,6 +346,7 @@ def parse_xpath_to_gnmi_path(self, xpath, origin=None): else: # module name origin, xpath = xpath.split(":", 1) + origin = origin.strip("/") return super(XRClient, self).parse_xpath_to_gnmi_path(xpath, origin) def parse_cli_to_gnmi_path(self, command): From b0ea34ce5186a7b0c449d82924e6d2a755d98379 Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 27 Apr 2020 18:59:52 -0700 Subject: [PATCH 21/27] Fix for check_config and prefix. --- src/cisco_gnmi/client.py | 14 +++++++++++++- src/cisco_gnmi/xe.py | 15 +++++++++------ 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index d3caf08..443219f 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -31,6 +31,9 @@ from . import util +logger = logging.getLogger(__name__) + + class Client(object): """gNMI gRPC wrapper client to ease usage of gNMI. @@ -163,6 +166,9 @@ def get( request.use_models = use_models if extension: request.extension = extension + + logger.debug(str(request)) + get_response = self.service.Get(request) return get_response @@ -190,7 +196,7 @@ def set( """ request = proto.gnmi_pb2.SetRequest() if prefix: - request.prefix = prefix + request.prefix.CopyFrom(prefix) test_list = [updates, replaces, deletes] if not any(test_list): raise Exception("At least update, replace, or delete must be specified!") @@ -207,6 +213,9 @@ def set( request.delete.extend(deletes) if extensions: request.extension.extend(extensions) + + logger.debug(str(request)) + response = self.service.Set(request) return response @@ -244,6 +253,9 @@ def validate_request(request): ) if extensions: subscribe_request.extensions.extend(extensions) + + logger.debug(str(subscribe_request)) + return subscribe_request response_stream = self.service.Subscribe( diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 6c9e525..783cd8e 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -30,6 +30,9 @@ from .client import Client, proto, util +logger = logging.getLogger(__name__) + + class XEClient(Client): """IOS XE-specific wrapper for gNMI functionality. Assumes IOS XE 16.12+ @@ -108,7 +111,7 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True): + def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True, prefix=None): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -132,15 +135,15 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru raise Exception("Must supply at least one set of configurations to method!") def check_configs(name, configs): - if isinstance(name, string_types): - logging.debug("Handling %s as JSON string.", name) + if isinstance(configs, string_types): + logger.debug("Handling %s as JSON string.", name) try: configs = json.loads(configs) except: raise Exception("{name} is invalid JSON!".format(name=name)) configs = [configs] - elif isinstance(name, dict): - logging.debug("Handling %s as already serialized JSON object.", name) + elif isinstance(configs, dict): + logger.debug("Handling %s as already serialized JSON object.", name) configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( @@ -171,7 +174,7 @@ def create_updates(name, configs): updates = create_updates("update_json_configs", update_json_configs) replaces = create_updates("replace_json_configs", replace_json_configs) - return self.set(updates=updates, replaces=replaces) + return self.set(prefix=prefix, updates=updates, replaces=replaces) def get_xpaths(self, xpaths, data_type="ALL", encoding="JSON_IETF"): """A convenience wrapper for get() which forms proto.gnmi_pb2.Path from supplied xpaths. From 7d1135ccc6ad7acbe147ed3b5925554da3fcf5e6 Mon Sep 17 00:00:00 2001 From: miott Date: Mon, 27 Apr 2020 19:02:24 -0700 Subject: [PATCH 22/27] Remove test for removed xpath_util. --- tests/test_xpath.py | 167 -------------------------------------------- 1 file changed, 167 deletions(-) delete mode 100644 tests/test_xpath.py diff --git a/tests/test_xpath.py b/tests/test_xpath.py deleted file mode 100644 index 6fd556f..0000000 --- a/tests/test_xpath.py +++ /dev/null @@ -1,167 +0,0 @@ -import json -from src.cisco_gnmi import xpath_util - - -def test_parse_xpath_to_gnmi_path(): - result = xpath_util.parse_xpath_to_gnmi_path( - '/acl/acl-sets/acl-set', - origin='openconfig' - ) - assert str(result) == GNMI_UPDATE_ACL_SET - - -def test_combine_configs(): - pass - - -def test_xpath_to_json(): - pass - - -def test_get_payload(): - result = xpath_util.get_payload(PARSE_XPATH_TO_GNMI[1]['update']) - xpath, config = result[0] - assert xpath == '/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries' - # json_val - cfg = json.loads(config) - assert cfg == { - 'acl-entry': [ - { - 'config': {'forwarding-action': 'DROP'}, - 'ipv4': {'config': {'destination-address': '20.20.20.1/32', - 'protocol': 'IP_TCP', - 'source-address': '10.10.10.10/32'} - }, - 'sequence-id': '10' - } - ] - } - - -def test_xml_path_to_path_elem(): - result = xpath_util.xml_path_to_path_elem(XML_PATH_LANGUAGE_1) - assert result == ( - {'oc-acl': 'openconfig-acl'}, # module - { # config - 'delete': [], - 'get': [], - 'replace': [], - 'update': [ - {'/acl/acl-sets/acl-set': {'name': 'testacl'}}, - {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}} - ] - }, - 'openconfig' # origin - ) - - -XML_PATH_LANGUAGE_1 = { - 'namespace': { - 'oc-acl': 'http://openconfig.net/yang/acl' - }, - 'nodes': [ - { - 'value': 'testacl', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/name', - 'edit-op': 'merge' - }, - { - 'value': 'ACL_IPV4', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set/type', - 'edit-op': 'merge' - }, - { - 'value': '10', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry/oc-acl:sequence-id', - 'edit-op': 'merge' - }, - { - 'value': '20.20.20.1/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:destination-address', - 'edit-op': 'merge' - }, - { - 'value': 'IP_TCP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:protocol', - 'edit-op': 'merge' - }, - { - 'value': '10.10.10.10/32', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:ipv4/oc-acl:config/oc-acl:source-address', - 'edit-op': 'merge' - }, - { - 'value': 'DROP', - 'xpath': '/oc-acl:acl/oc-acl:acl-sets/oc-acl:acl-set[name="testacl"][type="ACL_IPV4"]/oc-acl:acl-entries/oc-acl:acl-entry[sequence-id="10"]/oc-acl:actions/oc-acl:config/oc-acl:forwarding-action', - 'edit-op': 'merge' - } - ] -} - - -PARSE_XPATH_TO_GNMI = ( - {'oc-acl': 'openconfig-acl'}, # module - { # config - 'delete': [], - 'get': [], - 'replace': [], - 'update': [ - {'/acl/acl-sets/acl-set': {'name': 'testacl'}}, - {'/acl/acl-sets/acl-set': {'type': 'ACL_IPV4'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry': {'sequence-id': '10'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'destination-address': '20.20.20.1/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'protocol': 'IP_TCP'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/ipv4/config': {'source-address': '10.10.10.10/32'}}, - {'/acl/acl-sets/acl-set[name="testacl"][type="ACL_IPV4"]/acl-entries/acl-entry[sequence-id="10"]/actions/config': {'forwarding-action': 'DROP'}} - ] - }, - 'openconfig' # origin -) - - -GNMI_UPDATE_ACL_ENTRY = """[path { -origin: "openconfig" -elem { - name: "acl" -} -elem { - name: "acl-sets" -} -elem { - name: "acl-set" - key { - key: "name" - value: "testacl" - } - key { - key: "type" - value: "ACL_IPV4" - } -} -elem { - name: "acl-entries" -} -} -val { -json_val: "{\"acl-entry\": [{\"actions\": {\"config\": {\"forwarding-action\": \"DROP\"}}, \"ipv4\": {\"config\": {\"destination-address\": \"20.20.20.1/32\", \"protocol\": \"IP_TCP\", \"source-address\": \"10.10.10.10/32\"}}, \"sequence-id\": \"10\"}]}" -} -] -""" - - -GNMI_UPDATE_ACL_SET = """origin: "openconfig" -elem { - name: "acl" -} -elem { - name: "acl-sets" -} -elem { - name: "acl-set" -} -""" From a5ac35fd9259568aa607c2a09e1203edec7db4f0 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Thu, 30 Apr 2020 15:47:42 -0700 Subject: [PATCH 23/27] Add debug log to Capabilities --- src/cisco_gnmi/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 443219f..0ecd241 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -112,6 +112,7 @@ def capabilities(self): proto.gnmi_pb2.CapabilityResponse """ message = proto.gnmi_pb2.CapabilityRequest() + logger.debug(message) response = self.service.Capabilities(message) return response From d0237746d5e233c3ecabb8e29f7b0b1e2852f5f1 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Thu, 30 Apr 2020 16:35:42 -0700 Subject: [PATCH 24/27] Add logger to all modules --- src/cisco_gnmi/builder.py | 15 +++++++++------ src/cisco_gnmi/client.py | 10 +++++----- src/cisco_gnmi/nx.py | 3 +++ src/cisco_gnmi/util.py | 13 ++++++++----- src/cisco_gnmi/xe.py | 14 ++++++++++---- src/cisco_gnmi/xr.py | 7 +++++-- 6 files changed, 40 insertions(+), 22 deletions(-) diff --git a/src/cisco_gnmi/builder.py b/src/cisco_gnmi/builder.py index a74a219..8b14450 100644 --- a/src/cisco_gnmi/builder.py +++ b/src/cisco_gnmi/builder.py @@ -31,6 +31,9 @@ from .util import gen_target_netloc, get_cert_from_target, get_cn_from_cert +LOGGER = logging.getLogger(__name__) + + class ClientBuilder(object): """Builder for the creation of a gNMI client. Supports construction of base Client and XRClient. @@ -134,8 +137,8 @@ def set_os(self, name=None): if name not in self.os_class_map.keys(): raise Exception("OS not supported!") else: + LOGGER.debug("Using %s wrapper.", name or "Client") self.__client_class = self.os_class_map[name] - logging.debug("Using %s wrapper.", name or "Client") return self def set_secure( @@ -257,7 +260,7 @@ def set_channel_option(self, name, value): found_index = index break if found_index is not None: - logging.warning("Found existing channel option %s, overwriting!", name) + LOGGER.warning("Found existing channel option %s, overwriting!", name) self.__channel_options[found_index] = new_option else: self.__channel_options.append(new_option) @@ -279,18 +282,18 @@ def construct(self): self.__root_certificates, self.__private_key, self.__certificate_chain ) if self.__username and self.__password: + LOGGER.debug("Using username/password call authentication.") channel_metadata_creds = grpc.metadata_call_credentials( CiscoAuthPlugin(self.__username, self.__password) ) - logging.debug("Using username/password call authentication.") if channel_ssl_creds and channel_metadata_creds: + LOGGER.debug("Using SSL/metadata authentication composite credentials.") channel_creds = grpc.composite_channel_credentials( channel_ssl_creds, channel_metadata_creds ) - logging.debug("Using SSL/metadata authentication composite credentials.") else: + LOGGER.debug("Using SSL credentials, no metadata authentication.") channel_creds = channel_ssl_creds - logging.debug("Using SSL credentials, no metadata authentication.") if self.__ssl_target_name_override is not False: if self.__ssl_target_name_override is None: if not self.__root_certificates: @@ -298,7 +301,7 @@ def construct(self): self.__ssl_target_name_override = get_cn_from_cert( self.__root_certificates ) - logging.warning( + LOGGER.warning( "Overriding SSL option from certificate could increase MITM susceptibility!" ) self.set_channel_option( diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 0ecd241..aec65a5 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -31,7 +31,7 @@ from . import util -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class Client(object): @@ -112,7 +112,7 @@ def capabilities(self): proto.gnmi_pb2.CapabilityResponse """ message = proto.gnmi_pb2.CapabilityRequest() - logger.debug(message) + LOGGER.debug(message) response = self.service.Capabilities(message) return response @@ -168,7 +168,7 @@ def get( if extension: request.extension = extension - logger.debug(str(request)) + LOGGER.debug(str(request)) get_response = self.service.Get(request) return get_response @@ -215,7 +215,7 @@ def set( if extensions: request.extension.extend(extensions) - logger.debug(str(request)) + LOGGER.debug(str(request)) response = self.service.Set(request) return response @@ -255,7 +255,7 @@ def validate_request(request): if extensions: subscribe_request.extensions.extend(extensions) - logger.debug(str(subscribe_request)) + LOGGER.debug(str(subscribe_request)) return subscribe_request diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index 4aab05b..f6c67c0 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -30,6 +30,9 @@ from .client import Client, proto, util +LOGGER = logging.getLogger(__name__) + + class NXClient(Client): """NX-OS-specific wrapper for gNMI functionality. diff --git a/src/cisco_gnmi/util.py b/src/cisco_gnmi/util.py index 7d205a6..c002158 100644 --- a/src/cisco_gnmi/util.py +++ b/src/cisco_gnmi/util.py @@ -37,6 +37,9 @@ from urlparse import urlparse +LOGGER = logging.getLogger(__name__) + + def gen_target_netloc(target, netloc_prefix="//", default_port=9339): """Parses and validates a supplied target URL for gRPC calls. Uses urllib to parse the netloc property from the URL. @@ -51,11 +54,11 @@ def gen_target_netloc(target, netloc_prefix="//", default_port=9339): if not parsed_target.netloc: raise ValueError("Unable to parse netloc from target URL %s!" % target) if parsed_target.scheme: - logging.debug("Scheme identified in target, ignoring and using netloc.") + LOGGER.debug("Scheme identified in target, ignoring and using netloc.") target_netloc = parsed_target if parsed_target.port is None: ported_target = "%s:%i" % (parsed_target.hostname, default_port) - logging.debug("No target port detected, reassembled to %s.", ported_target) + LOGGER.debug("No target port detected, reassembled to %s.", ported_target) target_netloc = gen_target_netloc(ported_target) return target_netloc @@ -120,11 +123,11 @@ def get_cn_from_cert(cert_pem): cert_cns = cert_parsed.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME) if len(cert_cns) > 0: if len(cert_cns) > 1: - logging.warning( + LOGGER.warning( "Multiple CNs found for certificate, defaulting to the first one." ) cert_cn = cert_cns[0].value - logging.debug("Using %s as certificate CN.", cert_cn) + LOGGER.debug("Using %s as certificate CN.", cert_cn) else: - logging.warning("No CN found for certificate.") + LOGGER.warning("No CN found for certificate.") return cert_cn diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 783cd8e..25802ed 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -30,7 +30,7 @@ from .client import Client, proto, util -logger = logging.getLogger(__name__) +LOGGER = logging.getLogger(__name__) class XEClient(Client): @@ -111,7 +111,13 @@ def delete_xpaths(self, xpaths, prefix=None): paths.append(self.parse_xpath_to_gnmi_path(xpath)) return self.set(deletes=paths) - def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=True, prefix=None): + def set_json( + self, + update_json_configs=None, + replace_json_configs=None, + ietf=True, + prefix=None, + ): """A convenience wrapper for set() which assumes JSON payloads and constructs desired messages. All parameters are optional, but at least one must be present. @@ -136,14 +142,14 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru def check_configs(name, configs): if isinstance(configs, string_types): - logger.debug("Handling %s as JSON string.", name) + LOGGER.debug("Handling %s as JSON string.", name) try: configs = json.loads(configs) except: raise Exception("{name} is invalid JSON!".format(name=name)) configs = [configs] elif isinstance(configs, dict): - logger.debug("Handling %s as already serialized JSON object.", name) + LOGGER.debug("Handling %s as already serialized JSON object.", name) configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( diff --git a/src/cisco_gnmi/xr.py b/src/cisco_gnmi/xr.py index f1bc809..808fd73 100644 --- a/src/cisco_gnmi/xr.py +++ b/src/cisco_gnmi/xr.py @@ -30,6 +30,9 @@ from .client import Client, proto, util +LOGGER = logging.getLogger(__name__) + + class XRClient(Client): """IOS XR-specific wrapper for gNMI functionality. @@ -130,14 +133,14 @@ def set_json(self, update_json_configs=None, replace_json_configs=None, ietf=Tru def check_configs(name, configs): if isinstance(name, string_types): - logging.debug("Handling %s as JSON string.", name) + LOGGER.debug("Handling %s as JSON string.", name) try: configs = json.loads(configs) except: raise Exception("{name} is invalid JSON!".format(name=name)) configs = [configs] elif isinstance(name, dict): - logging.debug("Handling %s as already serialized JSON object.", name) + LOGGER.debug("Handling %s as already serialized JSON object.", name) configs = [configs] elif not isinstance(configs, (list, set)): raise Exception( From 73924f66c1261d8e78c08583a6d9fa20b0dc0595 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Thu, 30 Apr 2020 16:35:52 -0700 Subject: [PATCH 25/27] Bump version to 1.0.8 --- src/cisco_gnmi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cisco_gnmi/__init__.py b/src/cisco_gnmi/__init__.py index 60f494f..73dc717 100644 --- a/src/cisco_gnmi/__init__.py +++ b/src/cisco_gnmi/__init__.py @@ -30,4 +30,4 @@ from .xe import XEClient from .builder import ClientBuilder -__version__ = "1.0.7" +__version__ = "1.0.8" From ec75312bf5f6ae4bdb410209bba0eae25e150578 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Thu, 30 Apr 2020 16:39:40 -0700 Subject: [PATCH 26/27] Explicitly cast proto to str --- src/cisco_gnmi/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index aec65a5..14ba271 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -112,7 +112,7 @@ def capabilities(self): proto.gnmi_pb2.CapabilityResponse """ message = proto.gnmi_pb2.CapabilityRequest() - LOGGER.debug(message) + LOGGER.debug(str(message)) response = self.service.Capabilities(message) return response From 770d9e7c12106d4896c26ad0b029de23e6324ee6 Mon Sep 17 00:00:00 2001 From: Remington Campbell Date: Thu, 30 Apr 2020 16:47:00 -0700 Subject: [PATCH 27/27] Add logger alias for LOGGER --- src/cisco_gnmi/builder.py | 1 + src/cisco_gnmi/client.py | 1 + src/cisco_gnmi/nx.py | 1 + src/cisco_gnmi/util.py | 1 + src/cisco_gnmi/xe.py | 1 + src/cisco_gnmi/xr.py | 1 + 6 files changed, 6 insertions(+) diff --git a/src/cisco_gnmi/builder.py b/src/cisco_gnmi/builder.py index 8b14450..8fba394 100644 --- a/src/cisco_gnmi/builder.py +++ b/src/cisco_gnmi/builder.py @@ -32,6 +32,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER class ClientBuilder(object): diff --git a/src/cisco_gnmi/client.py b/src/cisco_gnmi/client.py index 14ba271..f4ec9d8 100755 --- a/src/cisco_gnmi/client.py +++ b/src/cisco_gnmi/client.py @@ -32,6 +32,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER class Client(object): diff --git a/src/cisco_gnmi/nx.py b/src/cisco_gnmi/nx.py index f6c67c0..ae6f729 100644 --- a/src/cisco_gnmi/nx.py +++ b/src/cisco_gnmi/nx.py @@ -31,6 +31,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER class NXClient(Client): diff --git a/src/cisco_gnmi/util.py b/src/cisco_gnmi/util.py index c002158..572a7d3 100644 --- a/src/cisco_gnmi/util.py +++ b/src/cisco_gnmi/util.py @@ -38,6 +38,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER def gen_target_netloc(target, netloc_prefix="//", default_port=9339): diff --git a/src/cisco_gnmi/xe.py b/src/cisco_gnmi/xe.py index 25802ed..ebd469e 100644 --- a/src/cisco_gnmi/xe.py +++ b/src/cisco_gnmi/xe.py @@ -31,6 +31,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER class XEClient(Client): diff --git a/src/cisco_gnmi/xr.py b/src/cisco_gnmi/xr.py index 808fd73..e7cfb4d 100644 --- a/src/cisco_gnmi/xr.py +++ b/src/cisco_gnmi/xr.py @@ -31,6 +31,7 @@ LOGGER = logging.getLogger(__name__) +logger = LOGGER class XRClient(Client):