upd harvester

- minimal demo
- minor fixes

Author: 7048730

README.md

@@ -177,5 +177,16 @@ Explanation of the switches:
 # (4) Harvester tool:
 
 Collects OAI-PMH metadata from catalogues, data registries and repositories and parses them to assist the completion of a metadata set such as the one in o2r.
+_Note, that this tool is currently only a demo._ 
 
-_TBD_
+
+	python o2rmeta.py harvest -e <ELEMENT> -q <QUERY>
+	
+Example call:
+	
+	python o2rmeta.py -debug harvest -e"doi" -q"10.14457/CU.THE.1989.1"
+
+Explanation of the switches:
+
++ `-e` <ELEMENT> : MD element type for search, e.g. _doi_ or _creator_
++ `-q` <QUERY> : MD content to start the search

broker/mappings/eudat_b2share-map.json

@@ -33,8 +33,9 @@
 		},
 		"open_access":  {
 			"translatesTo": "open_access",
+			"type": "boolean",
 			"hasParent": "root",
 			"needsParent": "root"
 		}
 	}
-}
+}

harvest/bases.json

@@ -55,6 +55,20 @@
 			"isSubscriberOnly": false,
 			"schema": ""
 		},
+		"oadoi": {
+			"url": "\\url{https://api.oadoi.org/}",
+			"default_parameter": "",
+			"isActive": true,
+			"isSubscriberOnly": false,
+			"schema": ""
+		},
+		"Orcid": {
+			"url": "\\url{https://pub.sandbox.orcid.org/v2.0/search?q=}",
+			"default_parameter": "",
+			"isActive": true,
+			"isSubscriberOnly": false,
+			"schema": ""
+		},
 		"OpenAIRE": {
 			"url": "\\url{http://api.openaire.eu/oai_pmh}",
 			"default_parameter": "",

harvest/metaharvest.py

@@ -1,38 +1,20 @@
-'''
-    Copyright (c) 2016 - o2r project
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-        http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-'''
-
-import argparse
 import base64
 import json
-import sys
 import urllib.request
 
 from lxml import etree
 
 
 def parse_from_unicode(unicode_str):
+    utf8_parser = etree.XMLParser(encoding='utf-8')
     s = unicode_str.encode('utf-8')
     return etree.fromstring(s, parser=utf8_parser)
 
 def qu(q_type,q_string,q_base):
     # check if json url has been parsed correctly
     if q_base.startswith('\\url{'):
         q_base = q_base[5:].replace('}', '')
-    accepted = ['doi','creator'] #<-- generify
+    accepted = ['doi', 'creator']
     if q_type in accepted:
         # datacite solr query encoding ideolect:
         my_query = str.encode('q='+q_type+'%3A'+q_string)
@@ -47,43 +29,39 @@ def qu(q_type,q_string,q_base):
         print('query type not available')
         return None
 
+
+def status_note(msg):
+    print(''.join(('[o2rmeta][harvest] ', str(msg))))
+
+
 # main:
-if __name__ == "__main__":
-    if sys.version_info[0] < 3:
-        # py2
-        print('requires py3k or later')
-        sys.exit()
-    else:
-        parser = argparse.ArgumentParser(description='description')
-        parser.add_argument('-i', '--input', help='type of provided metadata element for input, e.g. doi or creator', required=True)
-        parser.add_argument('-q', '--query', help='query string', required=True)
-        args = parser.parse_args()
-        argsdict = vars(args)
-        i = argsdict['input']
-        q = argsdict['query']
-        try:
-            with open('bases.json', encoding='utf-8') as data_file:
-                bases = json.load(data_file)
-                settings_data = bases['Settings']
-                baseurl_data = bases['BaseURLs']
-        except:
-            raise
-        print('[metaharvest] starting request')
-        #test datacite
-        my_base = 'DataCite' #<-- make this parsed arg
-        try:
-            result = qu(i.lower(), q, baseurl_data[my_base]['url'] + baseurl_data[my_base]['default_parameter'])
-            print('[metaharvest] !debug using ' + result[:128] + ' ...')
-            utf8_parser = etree.XMLParser(encoding='utf-8')
-            tree = parse_from_unicode(result)
-            # e.g. return author from datacite creatorName
-            output = {}
-            for node in tree.xpath('//ns:creatorName', namespaces={'ns': 'http://datacite.org/schema/kernel-3'}):
-                output['authorName'] = node.text
-            json_output = json.dumps(output)
-            #output if not empty
-            if str(json_output) != '{}':
-                print(str(json_output))
-        except:
-            raise
-            #pass
+def start(**kwargs):
+    global e
+    e = kwargs.get('e', None)
+    global q
+    q = kwargs.get('q', None)
+    try:
+        with open('harvest/bases.json', encoding='utf-8') as data_file:
+            bases = json.load(data_file)
+            settings_data = bases['Settings']
+            baseurl_data = bases['BaseURLs']
+    except:
+        raise
+    status_note('new request')
+    # demo datacite
+    my_base = 'DataCite'
+    # todo: argparse base endpoint
+    try:
+        result = qu(e.lower(), q, baseurl_data[my_base]['url'] + baseurl_data[my_base]['default_parameter'])
+        status_note('!debug using ' + result[:256] + ' ...')
+        tree = parse_from_unicode(result)
+        # e.g. return author from datacite creatorName
+        output = {}
+        for node in tree.xpath('//ns:creatorName', namespaces={'ns': 'http://datacite.org/schema/kernel-3'}):
+            output['authorName'] = node.text
+        json_output = json.dumps(output)
+        # output if not empty
+        if str(json_output) != '{}':
+            print(str(json_output))
+    except:
+        raise

o2rmeta.py

@@ -74,15 +74,17 @@ def status_note(msg, **kwargs):
         group.add_argument('-s', '--outputtostdout', help='output the result of the brokering to stdout',
                            action='store_true', default=False)
         # - - - - - - - - - - - - - - - - - -
+        harvester = subparsers.add_parser("harvest")
+        harvester.add_argument('-e', '--element', type=str,  help='element type, e.g. doi or creator',required=True)
+        harvester.add_argument('-q', '--query', type=str,  help='query string', required=True)
+        # - - - - - - - - - - - - - - - - - -
         validator = subparsers.add_parser("validate")
         validator.add_argument("-s", "--schema", type=str, required=True)
         validator.add_argument("-c", "--candidate", type=str, required=True)
         # - - - - - - - - - - - - - - - - - -
-        # harvester
-        # - - - - - - - - - - - - - - - - - -
         args = parser.parse_args()
         argsd = vars(args)
-        my_version = 23  # update me!
+        my_version = 24  # update me!
         my_mod = ''
         try:
             my_mod = datetime.datetime.fromtimestamp(os.stat(__file__).st_mtime)
@@ -102,7 +104,7 @@ def status_note(msg, **kwargs):
                 metavalidate.start(s=argsd['schema'], c=argsd['candidate'])
             elif argsd['tool'] == "harvest":
                 status_note('launching harvester')
-                print('TBD')  # todo
+                metaharvest.start(e=argsd['element'], q=argsd['query'])
             else:
                 pass
         except: