Changed parser action related code to allow single expressions

das/atomese_yacc_test.py

@@ -40,7 +40,7 @@ def test_action_broker():
     assert action_broker.count_terminal == 0
     assert action_broker.count_nested_expression == 0
     assert action_broker.count_toplevel_expression == 0
-    assert action_broker.count_type == 0
+    assert action_broker.count_type == 1
 
     action_broker = ActionBroker()
     yacc_wrap = AtomeseYacc(action_broker=action_broker)
@@ -49,7 +49,7 @@ def test_action_broker():
     assert action_broker.count_terminal == 11
     assert action_broker.count_nested_expression == 7
     assert action_broker.count_toplevel_expression == 4
-    assert action_broker.count_type == 9 + action_broker.count_terminal
+    assert action_broker.count_type == 10 + action_broker.count_terminal
 
     action_broker = ActionBroker(test_data)
     yacc_wrap = AtomeseYacc(action_broker=action_broker)
@@ -58,4 +58,4 @@ def test_action_broker():
     assert action_broker.count_terminal == 11
     assert action_broker.count_nested_expression == 7
     assert action_broker.count_toplevel_expression == 4
-    assert action_broker.count_type == 9 + action_broker.count_terminal
+    assert action_broker.count_type == 10 + action_broker.count_terminal

das/base_yacc.py

@@ -27,6 +27,7 @@
 import ply.yacc as yacc
 from das.expression_hasher import ExpressionHasher
 from das.expression import Expression
+from das.metta_lex import BASIC_TYPE
 
 class BaseYacc:
 
@@ -41,15 +42,28 @@ def __init__(self, **kwargs):
         self.pending_expression_names = []
         self.pending_named_types = []
         self.pending_expressions = []
-        self.named_types = {}
-        self.named_type_hash = {}
-        self.symbol_hash = {}
-        self.terminal_hash = {}
-        self.parent_type = {}
+        if kwargs.pop('use_action_broker_cache', False):
+            self.named_type_hash = self.action_broker.named_type_hash
+            self.named_types = self.action_broker.named_types
+            self.symbol_hash = self.action_broker.symbol_hash
+            self.terminal_hash = self.action_broker.terminal_hash
+            self.parent_type = self.action_broker.parent_type
+        else:
+            self.named_type_hash = {}
+            self.named_types = {}
+            self.symbol_hash = {}
+            self.terminal_hash = {}
+            self.parent_type = {}
+        basic_type_hash_code = ExpressionHasher._compute_hash(BASIC_TYPE)
+        self.named_type_hash[BASIC_TYPE] = basic_type_hash_code
+        self.parent_type[basic_type_hash_code] = basic_type_hash_code
 
     def setup(self):
         self.tokens = self.lex_wrap.tokens
         self.lexer = self.lex_wrap.lexer
+        if self.action_broker is not None:
+            expression = self._typedef(BASIC_TYPE, BASIC_TYPE)
+            self.action_broker.new_top_level_typedef_expression(expression)
 
     def _get_terminal_hash(self, named_type, terminal_name):
         key = (named_type, terminal_name)

das/das_update_test.py

@@ -0,0 +1,193 @@
+import tempfile
+import shutil
+import os
+import pytest
+from das.distributed_atom_space import DistributedAtomSpace, WILDCARD
+from das.database.db_interface import UNORDERED_LINK_TYPES
+
+das = DistributedAtomSpace()
+transaction = das.open_transaction()
+transaction.add_toplevel_expression('(: "human" Concept)')
+transaction.add_toplevel_expression('(Similarity "human" "monkey")')
+transaction.add_toplevel_expression('(: "gorilla" Concept)')
+transaction.add_toplevel_expression('(Inheritance "gorilla" "mammal")')
+transaction.add_toplevel_expression('(Similarity "gorilla" "human")')
+transaction.add_toplevel_expression('(Similarity "gorilla" "monkey")')
+transaction.add_toplevel_expression('(Similarity "gorilla" "chimp")')
+transaction.add_toplevel_expression('(Similarity "human" "gorilla")')
+transaction.add_toplevel_expression('(Similarity "monkey" "gorilla")')
+transaction.add_toplevel_expression('(Similarity "chimp" "gorilla")')
+das.commit_transaction(transaction)
+
+similarity = "Similarity"
+inheritance = "Inheritance"
+concept = "Concept"
+animal = das.get_node(concept, "animal")
+mammal = das.get_node(concept, "mammal")
+reptile = das.get_node(concept, "reptile")
+plant = das.get_node(concept, "plant")
+human = das.get_node(concept, "human")
+monkey = das.get_node(concept, "monkey")
+chimp = das.get_node(concept, "chimp")
+gorilla = das.get_node(concept, "gorilla")
+earthworm = das.get_node(concept, "earthworm")
+snake = das.get_node(concept, "snake")
+triceratops = das.get_node(concept, "triceratops")
+rhino = das.get_node(concept, "rhino")
+vine = das.get_node(concept, "vine")
+ent = das.get_node(concept, "ent")
+dinosaur = das.get_node(concept, "dinosaur")
+all_similarities = [
+    set([human, monkey]),
+    set([human, chimp]),
+    set([chimp, monkey]),
+    set([human, gorilla]),
+    set([monkey, gorilla]),
+    set([chimp, gorilla]),
+    set([earthworm, snake]),
+    set([triceratops, rhino]),
+    set([vine, snake]),
+    set([ent, human]),
+]
+all_inheritances = [
+    [human, mammal],
+    [monkey, mammal],
+    [chimp, mammal],
+    [gorilla, mammal],
+    [mammal, animal],
+    [reptile, animal],
+    [snake, reptile],
+    [dinosaur, reptile],
+    [triceratops, dinosaur],
+    [earthworm, animal],
+    [rhino, mammal],
+    [vine, plant],
+    [ent, plant],
+]
+
+template_similarity = [similarity, concept, concept]
+template_inheritance = [inheritance, concept, concept]
+
+def test_get_node():
+    human_document = das.get_node(concept, "human", build_node_dict=True)
+    assert human_document["handle"] == human
+    assert human_document["type"] == concept
+    assert human_document["name"] == "human"
+
+    gorilla_document = das.get_node(concept, "gorilla", build_node_dict=True)
+    assert gorilla_document["handle"] == gorilla
+    assert gorilla_document["type"] == concept
+    assert gorilla_document["name"] == "gorilla"
+
+def test_get_link():
+    link_handle = das.get_link(similarity, [human, monkey])
+    link = das.get_link(similarity, [human, monkey], build_link_dict = True)
+    assert link["handle"] == link_handle
+    assert link["type"] == similarity
+    assert link["template"] == template_similarity
+
+    link_handle = das.get_link(similarity, [human, gorilla])
+    link = das.get_link(similarity, [human, gorilla], build_link_dict = True)
+    assert link["handle"] == link_handle
+    assert link["type"] == similarity
+    assert link["template"] == template_similarity
+
+    link_handle = das.get_link(similarity, [gorilla, monkey])
+    link = das.get_link(similarity, [gorilla, monkey], build_link_dict = True)
+    assert link["handle"] == link_handle
+    assert link["type"] == similarity
+    assert link["template"] == template_similarity
+
+    link_handle = das.get_link(similarity, [gorilla, chimp])
+    link = das.get_link(similarity, [gorilla, chimp], build_link_dict = True)
+    assert link["handle"] == link_handle
+    assert link["type"] == similarity
+    assert link["template"] == template_similarity
+
+    link_handle = das.get_link(inheritance, [gorilla, mammal])
+    link = das.get_link(inheritance, [gorilla, mammal], build_link_dict = True)
+    assert link["handle"] == link_handle
+    assert link["type"] == inheritance
+    assert link["template"] == template_inheritance
+
+def test_get_links_with_link_templates():
+    link_handles = das.get_links(link_type=similarity, target_types=[concept, concept])
+    links = das.get_links(link_type=similarity, target_types=[concept, concept], build_link_dict=True)
+    assert len(link_handles) == len(links)
+    for link in links:
+        assert link["handle"] in link_handles
+        assert link["type"] == similarity
+        assert link["template"] == template_similarity
+        assert set(link["targets"]) in all_similarities
+
+def _check_pattern(link_type, targets, expected):
+    link_handles = das.get_links(link_type=link_type, targets=targets)
+    links = das.get_links(link_type=link_type, targets=targets, build_link_dict=True)
+    for link in links:
+        print(link)
+    assert len(link_handles) == len(links)
+    assert len(links) == len(expected)
+    for link in links:
+        assert link["handle"] in link_handles
+        assert link["type"] == link_type or link_type == WILDCARD
+        if link_type == similarity:
+            assert link["template"] == template_similarity
+        if link_type == inheritance:
+            assert link["template"] == template_inheritance
+        if link["type"] in UNORDERED_LINK_TYPES:
+            assert set(link["targets"]) in expected
+        else:
+            assert link["targets"] in expected
+
+def test_get_links_with_patterns():
+    #TODO: uncomment this check (and make it pass :-))
+    #_check_pattern(similarity, [WILDCARD, WILDCARD], all_similarities)
+    print("human", human)
+    print("monkey", monkey)
+    print("chimp", chimp)
+    print("ent", ent)
+    print("gorilla", gorilla)
+    _check_pattern(similarity, [human, WILDCARD], [
+        set([human, monkey]),
+        set([human, chimp]),
+        set([human, ent]),
+        set([human, gorilla]),
+    ])
+    _check_pattern(similarity, [WILDCARD, human], [
+        set([human, monkey]),
+        set([human, chimp]),
+        set([human, ent]),
+        set([human, gorilla]),
+    ])
+    _check_pattern(inheritance, [WILDCARD, WILDCARD], all_inheritances)
+    _check_pattern(inheritance, [human, WILDCARD], [
+        [human, mammal],
+    ])
+    _check_pattern(inheritance, [WILDCARD, animal], [
+        [mammal, animal],
+        [reptile, animal],
+        [earthworm, animal],
+    ])
+    _check_pattern(WILDCARD, [gorilla, monkey], [
+        set([gorilla, monkey]),
+    ])
+    _check_pattern(WILDCARD, [gorilla, chimp], [
+        set([chimp, gorilla]),
+    ])
+    _check_pattern(WILDCARD, [human, mammal], [
+        [human, mammal],
+    ])
+    _check_pattern(WILDCARD, [mammal, human], [
+    ])
+    _check_pattern(WILDCARD, [gorilla, mammal], [
+        [gorilla, mammal],
+    ])
+    _check_pattern(WILDCARD, [mammal, gorilla], [
+    ])
+    _check_pattern(WILDCARD, [human, WILDCARD], [
+        set([human, monkey]),
+        set([human, chimp]),
+        set([human, ent]),
+        set([human, gorilla]),
+        [human, mammal],
+    ])

das/database/couch_mongo_db.py

@@ -30,55 +30,66 @@ def __init__(self, couch_db: Bucket, mongo_db: Database):
         self.mongo_nodes_collection = self.mongo_db.get_collection(MongoCollectionNames.NODES)
         self.mongo_types_collection = self.mongo_db.get_collection(MongoCollectionNames.ATOM_TYPES)
         self.wildcard_hash = ExpressionHasher._compute_hash(WILDCARD)
-        self.node_handles = None
+        self.named_type_hash = None
+        self.named_type_hash_reverse = None
+        self.named_types = None
+        self.symbol_hash = None
+        self.terminal_hash = None
+        self.parent_type = None
         self.node_documents = None
-        self.atom_type_hash = None
-        self.atom_type_hash_reverse = None
         self.typedef_mark_hash = ExpressionHasher._compute_hash(":")
         self.typedef_base_type_hash = ExpressionHasher._compute_hash("Type")
         self.typedef_composite_type_hash = ExpressionHasher.composite_hash([
             self.typedef_mark_hash,
             self.typedef_base_type_hash,
             self.typedef_base_type_hash])
 
-    def _build_composite_node_name(self, node_type: str, node_name: str) -> str:
-        return f'{node_type}:{node_name}'
-
     def _get_atom_type_hash(self, atom_type):
         #TODO: implement a proper mongo collection to atom types so instead
         #      of this lazy hashmap, we should load the hashmap during prefetch
-        atom_type_hash = self.atom_type_hash.get(atom_type, None)
-        if atom_type_hash is None:
-            atom_type_hash = ExpressionHasher.named_type_hash(atom_type)
-            self.atom_type_hash[atom_type] = atom_type_hash
-            self.atom_type_hash_reverse[atom_type_hash] = atom_type
-        return atom_type_hash
+        named_type_hash = self.named_type_hash.get(atom_type, None)
+        if named_type_hash is None:
+            named_type_hash = ExpressionHasher.named_type_hash(atom_type)
+            self.named_type_hash[atom_type] = named_type_hash
+            self.named_type_hash_reverse[named_type_hash] = atom_type
+        return named_type_hash
 
     def _get_node_handle(self, node_type, node_name):
-        composite_name = self._build_composite_node_name(node_type, node_name)
-        node_handle = self.node_handles.get(composite_name, None)
+        composite_name = (node_type, node_name)
+        node_handle = self.terminal_hash.get(composite_name, None)
         if node_handle is None:
             node_handle = ExpressionHasher.terminal_hash(node_type, node_name)
-            self.node_handles[composite_name] = node_handle
+            self.terminal_hash[composite_name] = node_handle
         return node_handle
 
     def prefetch(self) -> None:
-        self.node_handles = {}
+        self.named_type_hash = {}
+        self.named_type_hash_reverse = {}
+        self.named_types = {}
+        self.symbol_hash = {}
+        self.terminal_hash = {}
+        self.parent_type = {}
         self.node_documents = {}
-        self.atom_type_hash = {}
-        self.atom_type_hash_reverse = {}
-        collection = self.mongo_nodes_collection
-        for document in collection.find():
+        for document in self.mongo_nodes_collection.find():
             node_id = document[MongoFieldNames.ID_HASH]
             node_type = document[MongoFieldNames.TYPE_NAME]
             node_name = document[MongoFieldNames.NODE_NAME]
             self.node_documents[node_id] = document
-            self.node_handles[self._build_composite_node_name(node_type, node_name)] = node_id
+            self.terminal_hash[(node_type, node_name)] = node_id
         for document in self.mongo_types_collection.find():
+            hash_id = document[MongoFieldNames.ID_HASH]
             named_type = document[MongoFieldNames.TYPE_NAME]
-            named_type_hash = document["named_type_hash"]
-            self.atom_type_hash[named_type] = named_type_hash
-            self.atom_type_hash_reverse[named_type_hash] = named_type
+            named_type_hash = document[MongoFieldNames.TYPE_NAME_HASH]
+            composite_type_hash = document[MongoFieldNames.TYPE]
+            type_document = self.mongo_types_collection.find_one({
+                MongoFieldNames.ID_HASH: composite_type_hash
+            })
+            self.named_type_hash[named_type] = named_type_hash
+            self.named_type_hash_reverse[named_type_hash] = named_type
+            if type_document is not None:
+                self.named_types[named_type] = type_document[MongoFieldNames.TYPE_NAME]
+                self.parent_type[named_type_hash] = type_document[MongoFieldNames.TYPE_NAME_HASH]
+            self.symbol_hash[named_type] = hash_id
 
     def _retrieve_mongo_document(self, handle: str, arity=-1) -> dict:
         mongo_filter = {MongoFieldNames.ID_HASH: handle}
@@ -124,7 +135,7 @@ def _build_named_type_hash_template(self, template: Union[str, List[Any]]) -> Li
 
     def _build_named_type_template(self, template: Union[str, List[Any]]) -> List[Any]:
         if isinstance(template, str):
-            return self.atom_type_hash_reverse.get(template, None)
+            return self.named_type_hash_reverse.get(template, None)
         else:
             answer = []
             for element in template:

das/database/couch_mongo_db_test.py

@@ -78,9 +78,12 @@ def test_db_creation(db: DBInterface):
     assert db.couch_outgoing_collection
     assert db.couch_patterns_collection
     assert len(db.node_documents) == 14
-    assert len(db.node_handles) == 14
-    print(db.atom_type_hash)
-    assert len(db.atom_type_hash) == 17
+    assert len(db.terminal_hash) == 14
+    assert len(db.named_type_hash) == 18
+    assert len(db.named_type_hash_reverse) == 18
+    assert len(db.named_types) == 18
+    assert len(db.symbol_hash) == 18
+    assert len(db.parent_type) == 18
 
 def test_node_exists(db: DBInterface):
     assert db.node_exists('Concept', 'human')
@@ -103,7 +106,7 @@ def test_node_exists(db: DBInterface):
 def _check_link(db: DBInterface, handle: str, link_type: str, target1: str, target2: str):
     collection = db.mongo_db.get_collection(MongoCollectionNames.LINKS_ARITY_2)
     document = collection.find_one({'_id': handle})
-    type_handle = db.atom_type_hash[link_type]
+    type_handle = db.named_type_hash[link_type]
     assert document
     assert document['named_type_hash'] == type_handle
     assert document['key_0'] == target1

das/database/mongo_schema.py

@@ -10,6 +10,7 @@ class CollectionNames(str, Enum):
 class FieldNames(str, Enum):
     NODE_NAME = 'name'
     TYPE_NAME = 'named_type'
+    TYPE_NAME_HASH = 'named_type_hash'
     ID_HASH = '_id'
     TYPE = 'composite_type_hash'
     COMPOSITE_TYPE = 'composite_type'