validator: Rework selecting schemas for validation

Validation of devicetrees is slow because every schema is checked
whether to apply it or not to every node. With 1000s of schemas and
100s-1000s of nodes in 1000s of DTBs, that makes for long validation
times. The reality is there's a few schemas that apply to all nodes
(mostly the core schemas) and only 1 schema with a matching compatible
string that applies (not counting schemas included by $ref). There's
also a few corner cases which have their own 'select'.

Rework the validation to take advantage of this by storing up front the
list of schemas to always iterate thru, and a map of compatible strings
to their respective schema. The result is a 6-7x reduction in the
validation time.

Signed-off-by: Rob Herring <[email protected]>

dtschema/fixups.py

@@ -405,46 +405,21 @@ def add_select_schema(schema):
     '''Get a schema to be used in select tests.
 
     If the provided schema has a 'select' property, then use that as the select schema.
-    If it has a compatible property, then create a select schema from that.
     If it has a $nodename property, then create a select schema from that.
-    If it has none of those, then return a match-nothing schema
     '''
     if "select" in schema:
         return
 
-    if 'properties' not in schema:
-        schema['select'] = False
+    if 'properties' not in schema or 'compatible' in schema['properties']:
         return
 
-    if 'compatible' in schema['properties']:
-        compatible_list = dtschema.extract_node_compatibles(schema['properties']['compatible'])
-
-        if len(compatible_list):
-            try:
-                compatible_list.remove('syscon')
-            except:
-                pass
-            try:
-                compatible_list.remove('simple-mfd')
-            except:
-                pass
-
-            if len(compatible_list) != 0:
-                schema['select'] = {
-                    'required': ['compatible'],
-                    'properties': {'compatible': {'contains': {'enum': sorted(compatible_list)}}}}
-
-                return
-
     if '$nodename' in schema['properties'] and schema['properties']['$nodename'] is not True:
         schema['select'] = {
             'required': ['$nodename'],
             'properties': {'$nodename': convert_to_dict(schema['properties']['$nodename'])}}
 
         return
 
-    schema['select'] = False
-
 
 def fixup_schema(schema):
     # Remove parts not necessary for validation

dtschema/validator.py

@@ -239,7 +239,7 @@ def make_compatible_schema(schemas):
 
     compat_sch[0]['enum'].sort()
     schemas['generated-compatibles'] = {
-        '$id': 'http://devicetree.org/schemas/generated-compatibles',
+        '$id': 'generated-compatibles',
         '$filename': 'Generated schema of documented compatible strings',
         'select': True,
         'properties': {
@@ -269,9 +269,6 @@ def process_schema(filename):
         return
 
     schema = dtsch.fixup()
-    if 'select' not in schema:
-        print(f"{filename}: warning: no 'select' found in schema found", file=sys.stderr)
-        return
 
     schema["type"] = "object"
     schema["$filename"] = filename
@@ -378,6 +375,20 @@ def __init__(self, schema_files, filter=None):
             for k in self.pat_props:
                 self.pat_props[k][0]['regex'] = re.compile(k)
 
+        # Speed up iterating thru schemas in validation by saving a list of schemas
+        # to always apply and a map of compatible strings to schema.
+        self.always_schemas = []
+        self.compat_map = {}
+        for sch in self.schemas.values():
+            if 'select' in sch:
+                if sch['select'] is not False:
+                    self.always_schemas += [sch['$id']]
+            elif 'properties' in sch and 'compatible' in sch['properties']:
+                compatibles = dtschema.extract_node_compatibles(sch['properties']['compatible'])
+                compatibles = set(compatibles) - {'syscon', 'simple-mfd'}
+                for c in compatibles:
+                    self.compat_map[c] = sch['$id']
+
         self.schemas['version'] = dtschema.__version__
 
     def http_handler(self, uri):
@@ -399,16 +410,26 @@ def annotate_error(self, id, error):
         error.note = None
 
     def iter_errors(self, instance, filter=None):
-        for id, schema in self.schemas.items():
-            if 'select' not in schema:
-                continue
-            if filter and filter not in id:
+        if 'compatible' in instance:
+            inst_compat = instance['compatible'][0]
+            if inst_compat in self.compat_map:
+                schema_id = self.compat_map[inst_compat]
+                if not filter or filter in schema_id:
+                    schema = self.schemas[schema_id]
+                    for error in self.DtValidator(schema,
+                                                  resolver=self.resolver,
+                                                  ).iter_errors(instance):
+                        self.annotate_error(schema['$id'], error)
+                        yield error
+        for schema_id in self.always_schemas:
+            if filter and filter not in schema_id:
                 continue
-            sch = {'if': schema['select'], 'then': schema}
-            for error in self.DtValidator(sch,
+            schema = {'if': self.schemas[schema_id]['select'],
+                      'then': self.schemas[schema_id]}
+            for error in self.DtValidator(schema,
                                           resolver=self.resolver,
                                           ).iter_errors(instance):
-                self.annotate_error(id, error)
+                self.annotate_error(schema_id, error)
                 yield error
 
     def validate(self, instance, filter=None):