Merge pull request #3 from HTTP-APIs/master

Sync with upstream

dockerfile → Dockerfile

@@ -10,3 +10,6 @@ COPY ./hydra_redis /home/app/hydra_redis
 ENV PYTHONPATH $PYTHONPATH:/home/app/
 
 ENTRYPOINT ["python", "/home/app/hydra_redis/querying_mechanism.py"]
+
+
+

README.md

@@ -1,32 +1,40 @@
 # python-hydra-agent
 
-The python-hydra-agent is a smart python hydra client which is working with [hydrus](https://github.com/HTTP-APIs/hydrus).
+For a general introduction to Hydra Ecosystem, see [hydraecosystem.org](http://hydraecosystem.org).
 
-It caches the server data from hydra server for fast data querying.
+`python-hydra-agent` is a smart Hydra client implemented in Python which works with [hydrus](https://github.com/HTTP-APIs/hydrus). Reference implementation is [Heracles.ts](https://github.com/HydraCG/Heracles.ts). Smart clients are generic automated clients that establish resilient connected data networks leveraging knowledge graphs.
 
-It uses Redis to cache the data at the end of the client.
+## General characteristics
 
-So, Data is loaded from the server and store in Redis memory as a graph using redisgraph.
+The client is designed to:
+* Cache metadata from the Hydra server it connects to, to allow querying on the client-side;
+* Use Redis as a graph-store leveraging `redisgraph` (see [here](https://oss.redislabs.com/redisgraph/));
+* simply, metadata and data are loaded from the server and stored in Redis;
+* The graph can be queried using OpenCypher.
 
-With the help of Redis, clients become faster and easier to query the data.
+The starting objective is to create a querying layer that is able to reach data in one or more Hydra srever/s. Leveraging Redis, clients can construct their own representation of the data stored in one or more Hydra servers; querying the data as they need it, and respond complex semantic queries. This will allow any client connected to any server to have access to an "aggregated view" of the connected network (the network of all the servers it connects to). 
+
+## Missing bits at the moment
+* For now it is a proof-of-concept, only `GET` functionality
+* Soon to develop, a reliable synchronization mechanism to allow strong consistency between server-side data and client-side representation.
 
 ## Installation
 
 **NOTE:** You'll need to use python3.
 
+To install only requirements:
+
+    pip3 install -r requirements.txt
+
+or,
+
 To install or setup the client environment, you have to run:
 
      python3 setup.py install
 
-or,
 
-To install only requirements:
-
-    pip3 install -r requirements.txt
-
-To install Redis and start Redis server:
+To install Redis and other Redis modules:
 
-    cd hydra_redis
     ./redis_setup.sh
 
 ## Quickstart
@@ -48,17 +56,16 @@ To run the demo for python-hydra-agent, you have to follow the instructions:
 
     you should follow the instructions of [installation](#installation).
 
-After setup the environment and run the Redis server. You can query or run the client. 
+After setup the environment. You can query or run the client.
 
-* To run the client you should run querying_mechanism.py like:
-
-        cd hydra_redis
-        python3 querying_mechanism.py
+* To run both the things Redis server and the client. You can run the command:
+
+        docker-compose run client
 
 
     and provide a valid URL and then you can query in querying format.
 
-        `>>>url` #here url should be a valid link, for testing you can use https://storage.googleapis.com/api3/api
+        `>>>url` #here url should be a valid link, for testing you can use http://35.224.198.158:8080/api
         `>>>help` # it will provide the querying format
 
 #### Code simplification
@@ -96,18 +103,24 @@ The client takes the query as input, like:
 you can query as following querying formats:
 
 ```
-    print("for endpoint:- show endpoint")
-    print("for class_endpoint:- show classEndpoint")
-    print("for collection_endpoint:- show collectionEndpoint")
-    print("for members of collection_endpoint:-",
+    print("querying format")
+    print("Get all endpoints:- show endpoints")
+    print("Get all class_endpoints:- show classEndpoints")
+    print("Get all collection_endpoints:- show collectionEndpoints")
+    print("Get all members of collection_endpoint:-",
           "show <collection_endpoint> members")
-    print("for properties of any member:-",
+    print("Get all properties of objects:-",
+          "show objects<endpoint_type> properties")
+    print("Get all properties of any member:-",
           "show object<id_of_member> properties ")
-    print("for properties of objects:-show objects<endpoint_type> properties")
-    print("for collection properties:-",
-          "show <collection_endpoint> properties")
-    print("for classes properties:- show class<class_endpoint> properties")
-    print("for compare properties:-show <key> <value> and/or <key1> <value1>")
+    print("Get all classes properties:-show class<class_endpoint> properties")
+    print("Get data with compare properties:-",
+          "show <key> <value> and/or <key1> <value1>")
+    print("Get data by using both opeartions(and,or)",
+          " you should use brackets like:-",
+          "show model xyz and (name Drone1 or name Drone2)",
+          "or, show <key> <value> and (<key> <value> or <key> <value>)")
+
 ```
 
 Query test can be done like this:

hydra_redis/classes_objects.py

@@ -1,7 +1,15 @@
 import urllib.request
 import json
+import logging
 from redisgraph import Node, Edge
+from urllib.error import URLError, HTTPError
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class RequestError(Exception):
+    """A class for client-side exceptions"""
+    pass
 
 class ClassEndpoints:
     """Contains all the classes endpoint and the objects"""
@@ -119,8 +127,19 @@ def load_from_server(
         # new_url is url for the classes endpoint
         new_url = base_url + "/" + endpoint
         # retreiving data for the classes endpoint from server
-        response = urllib.request.urlopen(new_url)
-        new_file = json.loads(response.read().decode('utf-8'))
+        try:
+            response = urllib.request.urlopen(new_url)
+        except HTTPError as e:
+            logger.info('Error code: ', e.code)
+            return None
+        except URLError as e:
+            logger.info('Reason: ', e.reason)
+            return None
+        except ValueError as e:
+            logger.info("value error:", e)
+            return None
+        else:
+            new_file = json.loads(response.read().decode('utf-8'))
         # endpoint_property store all properties which is class/object but not
         # endpoint.
         for support_property in api_doc.parsed_classes[
@@ -161,7 +180,12 @@ def load_from_server(
                 endpoint_property,
                 no_endpoint_property,
                 api_doc)
-        # save the graph changes.
+        # delete all the old data that has saved in Redis using redis_graph.
+        # It will remove duplicate data from Redis.
+        for key in redis_connection.keys():
+            if "fs:" not in key.decode("utf8"):
+                redis_connection.delete(key)
+        # save the new data.
         self.redis_graph.commit()
 
     def endpointclasses(

hydra_redis/collections_endpoint.py

@@ -1,8 +1,12 @@
 import urllib.request
 import json
 import re
-from hydra_redis.classes_objects import ClassEndpoints
+import logging
+from urllib.error import URLError, HTTPError
+from hydra_redis.classes_objects import ClassEndpoints,RequestError
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
 class CollectionEndpoints:
     """Contains all the collections endpoints and objects"""
@@ -16,8 +20,19 @@ def fetch_data(self, new_url):
         :param new_url: url for fetching the data.
         :return: loaded data.
         """
-        response = urllib.request.urlopen(new_url)
-        return json.loads(response.read().decode('utf-8'))
+        try:
+            response = urllib.request.urlopen(new_url)
+        except HTTPError as e:
+            logger.info('Error code: ', e.code)
+            return RequestError("error")
+        except URLError as e:
+            logger.info('Reason: ', e.reason)
+            return RequestError("error")
+        except ValueError as e:
+            logger.info("value error:",e)
+            return RequestError("error")
+        else:
+            return json.loads(response.read().decode('utf-8'))
 
     def faceted_key(self, fs, key, value):
         return ("{}".format(fs + ":" + key + ":" + value))
@@ -68,6 +83,8 @@ def collectionobjects(
                 member_url = new_url + "/" + member_id
                 # object data retrieving from the server
                 new_file = self.fetch_data(member_url)
+                if isinstance (new_file, RequestError):
+                    return None
                 for support_operation in api_doc.parsed_classes[
                     endpoint["@type"]
                 ]["class"
@@ -161,6 +178,8 @@ def load_from_server(
         new_url = url + "/" + endpoint
         # url for every collection endpoint
         new_file = self.fetch_data(new_url)
+        if isinstance (new_file, RequestError):
+            return None
         # retrieving the objects from the collection endpoint
         for node in self.redis_graph.nodes.values():
             if node.alias == endpoint:
@@ -176,6 +195,12 @@ def load_from_server(
             url,
             redis_connection
         )
+        # delete all the old data that has saved in Redis using redis_graph.
+        # It will remove duplicate data from Redis.
+        for key in redis_connection.keys():
+            if "fs:" not in key.decode("utf8"):
+                redis_connection.delete(key)
+        # save the new data.
         self.redis_graph.commit()
 #        for node in self.redis_graph.nodes.values():
 #            print("\n",node.alias)

hydra_redis/hydra_graph.py

@@ -5,23 +5,14 @@
 from hydrus.hydraspec import doc_maker
 import hydrus
 from graphviz import Digraph
-from hydra_redis.classes_objects import ClassEndpoints
+from hydra_redis.classes_objects import ClassEndpoints,RequestError
 from hydra_redis.collections_endpoint import CollectionEndpoints
 from hydra_redis.redis_proxy import RedisProxy
 
 
 class InitialGraph:
 
 
-    def final_file(self,url):
-        """Open the given url and read and load the Json data.
-        :param url: given url to access the data from the server.
-        :return: data loaded from the server.
-        """
-        response = urllib.request.urlopen(url)
-        return json.loads(response.read().decode('utf-8'))
-
-
     def get_apistructure(self,entrypoint_node, api_doc):
         """ It breaks the endpoint into two parts collection and classes"""
         self.collection_endpoints = {}
@@ -68,17 +59,19 @@ def get_endpoints(self,api_doc, redis_connection):
         return self.get_apistructure(entrypoint_node, api_doc)
 
 
-    def main(self,new_url,api_doc):
+
+    def main(self,new_url,api_doc,check_commit):
         redis_connection = RedisProxy()
         redis_con = redis_connection.get_connection()
         self.url = new_url
         self.redis_graph = Graph("apidoc", redis_con)
         print("loading... of graph")
         self.get_endpoints(api_doc, redis_con)
-        print("commiting")
-        self.redis_graph.commit()
-        # creating whole the graph in redis
-        print("done!!!!")
+        if check_commit:
+            print("commiting")
+            self.redis_graph.commit()
+            # creating whole the graph in redis
+            print("done!!!!")
         # uncomment below 2 lines for getting nodes for whole graph
     #    for node in redis_graph.nodes.values():
     #        print("\n",node.alias)