upd extract & broker

- fix description handling
- upd broker input to take file path instead of dir
- minor fixes
- upd readme

Author: 7048730

.gitignore

@@ -4,6 +4,8 @@ broker/tests/all
 metadata_raw.json
 metadata_o2r.json
 metadata_zenodo.json
+metadata_eudat.json
+metadata_mods.json
 testerc.yml
 erc.yml
 erc_raw.yml

README.md

@@ -31,6 +31,7 @@ o2r meta is designed for python 3.6 and supports python 3.4+.
     sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y
     sudo apt-get -qq update
     sudo apt-get install -y python3-dev
+    sudo apt-get install -y libgdal1h
     sudo apt-get install -y libgdal-dev
     sudo apt-get build-dep -y python-gdal
     sudo apt-get install -y python-gdal
@@ -128,20 +129,20 @@ In _checking mode_ it returns missing metadata information for a target service
 Within o2r, the broker is used to translates between different standards for metadata sets. For example from extracted raw metadata to the o2r schema-compliant metadata. Other target outputs might DataCite XML or Zenodo JSON.
 Translation instructions as well as checklists are stored in json formatted files.
 
-    python o2rmeta.py broker -i <INPUT_DIR/FILE> -c <CHECKLIST_FILE>|-m <MAPPING_FILE> -s|-o <OUTPUT_DIR>
+    python o2rmeta.py broker -i <INPUT_FILE> -c <CHECKLIST_FILE>|-m <MAPPING_FILE> -s|-o <OUTPUT_DIR>
 
 Example calls:
 
     python o2rmeta.py -debug broker -c broker/checks/zenodo-check.json -i schema/json/example_zenodo.json -o broker/tests/all
 
-    python o2rmeta.py -debug broker -m broker/mappings/zenodo-map.json -i broker/tests -o broker/tests/all
+    python o2rmeta.py -debug broker -m broker/mappings/zenodo-map.json -i broker/tests/metadata_raw.json -o broker/tests/all
 
 Explanation of the switches:
 
 
 + `-c` <CHECKLIST_FILE> : required path to a json checklist file that holds checking instructions for the metadata. This switch is mutually exclusive with `-m`. At least one of them must be given.
 + `-m` <MAPPING_FILE> : required path to a json mapping file that holds translation instructions for the metadata mappings. This switch is mutually exclusive with `-c`. At least one of them must be given.
-+ `-i` <INPUT_DIR> : path to input json when using `-c`-mode _or_ required starting path for recursive search for parsable files when using `-m`-mode. 
++ `-i` <INPUT_FILE> : path to input json file.
 + `-s`: option to print out results to console. This switch is mutually exclusive with `-o`. At least one of them must be given.
 + `-o` <OUTPUT_DIR> : required output path, where data should be saved. If the directory does not exist, it will be created on runtime. This switch is mutually exclusive with `-s`. At least one of them must be given.
 

broker/metabroker.py

@@ -1,5 +1,5 @@
 """
-    Copyright (c) 2016 - o2r project
+    Copyright (c) 2016, 2017 - o2r project
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -31,23 +31,21 @@ def check(checklist_pathfile, input_json):
     # prepare input filepath
     try:
         if os.path.isfile(input_json):
-            ##print("is file")
             with open(input_json, encoding='utf-8') as data_file:
                 input_data = json.load(data_file)
-        # open checklist file and find out mode
-        output_dict = {'required': []}
-        with open(checklist_pathfile, encoding='utf-8') as data_file:
-            check_file = json.load(data_file)
-            settings_data = check_file['Settings']  # json or xml
-            checklist_data = check_file['Checklist']
-            #my_mode = settings_data['mode']
-            # todo:
-            #check_data_conditions = check_file['Conditions']
-            for x in checklist_data:
-                if x not in input_data:
-                    output_dict['required'].append(x)
-            do_outputs(output_dict, output_dir, settings_data['outputfile'])
-
+            # open checklist file and find out mode
+            output_dict = {'required': []}
+            with open(checklist_pathfile, encoding='utf-8') as data_file:
+                check_file = json.load(data_file)
+                settings_data = check_file['Settings']  # json or xml
+                checklist_data = check_file['Checklist']
+                #my_mode = settings_data['mode']
+                # todo:
+                #check_data_conditions = check_file['Conditions']
+                for x in checklist_data:
+                    if x not in input_data:
+                        output_dict['required'].append(x)
+                do_outputs(output_dict, output_dir, settings_data['outputfile'])
     except:
         raise
 
@@ -66,12 +64,12 @@ def do_outputs(output_data, out_mode, out_name):
             if not os.path.exists(out_mode):
                 os.makedirs(out_mode)
             with open(output_filename, 'w', encoding='utf-8') as outfile:
-                # todo: add handling if mode is txt, xml, ...
                 # for json:
                 output_data = json.dumps(output_data, sort_keys=True, indent=4, separators=(',', ': '))
                 outfile.write(str(output_data))
-            status_note(''.join(
-                (str(os.stat(output_filename).st_size), ' bytes written to ', os.path.abspath(output_filename))))
+                # for xml:
+                # TBD
+            status_note(''.join((str(os.stat(output_filename).st_size), ' bytes written to ', os.path.abspath(output_filename))))
         except Exception as exc:
             status_note(''.join(('! error while creating outputs: ', exc.args[0])))
 
@@ -113,7 +111,6 @@ def map_json(element, value, map_data, output_dict):
         for key in value:
             # ---<key:string>----------------------------------------------
             if type(key) is str:
-
                 if key in map_data:
                     d = 0
                     # ---<subkey:string>----------------------------------------------
@@ -153,7 +150,7 @@ def map_json(element, value, map_data, output_dict):
                             output_dict[map_data[y]['needsParent']].append(value[c][y])
             # ---<key:dict>----------------------------------------------
             elif type(key) is dict:
-                # as for 'authors'
+                # e.g. for 'authors'
                 location = ''
                 temp = {}
                 if type(key) is dict:
@@ -173,6 +170,7 @@ def map_json(element, value, map_data, output_dict):
 
 
 def map_xml(element, value, map_data, xml_root):
+    seperator = '#'
     a = None
     try:
         if type(value) is list or type(value) is dict:
@@ -248,12 +246,11 @@ def status_note(msg):
 
 # Main
 def start(**kwargs):
-    global input_dir
-    input_dir = kwargs.get('i', None)
+    global input_file
+    input_file = kwargs.get('i', None)
     global output_dir
     output_dir = kwargs.get('o', None)
     output_to_console = kwargs.get('s', None)
-    seperator = '#' #<-- make this generic
     global my_check
     my_check = kwargs.get('c', None)
     global my_map
@@ -269,7 +266,7 @@ def start(**kwargs):
         # not possible currently because output arg group is on mutual exclusive
         output_mode = '@none'
     if my_check is not None:
-        check(my_check, input_dir)
+        check(my_check, input_file)
     if my_map is not None:
         # open map file and find out mode
         try:
@@ -282,18 +279,18 @@ def start(**kwargs):
             raise
         # distinguish format for output
         if my_mode == 'json':
-            # try parse all possible metadata files:
-            for file in os.listdir(input_dir):
-                if os.path.basename(file).startswith('metadata_'):
-                    json_output = {}
-                    with open(os.path.join(input_dir, file), encoding='utf-8') as data_file:
-                        test_data = json.load(data_file)
-                    for element in test_data:
-                        try:
-                            map_json(element, test_data[element], map_data, json_output)
-                        except:
-                            raise
-                    do_outputs(json_output, output_mode, settings_data['outputfile'])
+            # parse target file # try parse all possible metadata files:
+            if not os.path.basename(input_file).startswith('metadata_'):
+                status_note('Warning: inputfile does not look like a metadata file object')
+            json_output = {}
+            with open(os.path.join(input_file), encoding='utf-8') as data_file:
+                test_data = json.load(data_file)
+            for element in test_data:
+                try:
+                    map_json(element, test_data[element], map_data, json_output)
+                except:
+                    raise
+            do_outputs(json_output, output_mode, settings_data['outputfile'])
         elif my_mode == 'txt':
             # to do: handle txt based maps like bagit
             txt_output = ''

extract/metaextract.py

@@ -188,8 +188,14 @@ def parse_yaml(input_text):
     # This is for R markdown files with yaml headers
     try:
         yaml_data_dict = yaml.safe_load(input_text)
-        # get authors and possible ids // orcid
         if yaml_data_dict is not None:
+            # model description / abstract:
+            if 'description' in yaml_data_dict:
+                if yaml_data_dict['description'] is not None:
+                    MASTER_MD_DICT['description'] = yaml_data_dict['description']
+            else:
+                if 'abstract' in yaml_data_dict:
+                    MASTER_MD_DICT['description'] = yaml_data_dict['abstract']
             # model author:
             if 'author' in yaml_data_dict:
                 if type(yaml_data_dict['author']) is str:
@@ -653,7 +659,6 @@ def start(**kwargs):
             status_note(''.join(('processing ', os.path.join(root, file).replace('\\', '/'))), b=log_buffer)
             # new file / new source
             nr += 1
-
             # interact with different file formats:
             if file_extension == '.txt':
                 if file.lower() == 'bagit.txt':

o2rmeta.py

@@ -68,7 +68,7 @@ def status_note(msg, **kwargs):
         maingroup = broker.add_mutually_exclusive_group(required=True)
         maingroup.add_argument("-c", "--check", type=str, required=False)
         maingroup.add_argument("-m", "--map", type=str, required=False)
-        broker.add_argument("-i", "--inputdir", type=str, required=True)
+        broker.add_argument("-i", "--inputfile", type=str, required=True)
         group = broker.add_mutually_exclusive_group(required=True)
         group.add_argument('-o', '--outputdir', type=str, help='output directory for brokering docs')
         group.add_argument('-s', '--outputtostdout', help='output the result of the brokering to stdout',
@@ -84,7 +84,7 @@ def status_note(msg, **kwargs):
         # - - - - - - - - - - - - - - - - - -
         args = parser.parse_args()
         argsd = vars(args)
-        my_version = 25  # update me!
+        my_version = 26  # update me!
         my_mod = ''
         try:
             my_mod = datetime.datetime.fromtimestamp(os.stat(__file__).st_mtime)
@@ -98,7 +98,7 @@ def status_note(msg, **kwargs):
                 metaextract.start(i=argsd['inputdir'], o=argsd['outputdir'], s=argsd['outputtostdout'], xo=argsd['stayoffline'], e=argsd['ercid'], m=argsd['metafiles'], xml=argsd['modexml'])
             elif argsd['tool'] == "broker":
                 status_note('launching broker')
-                metabroker.start(c=argsd['check'], m=argsd['map'], i=argsd['inputdir'], o=argsd['outputdir'], s=argsd['outputtostdout'])
+                metabroker.start(c=argsd['check'], m=argsd['map'], i=argsd['inputfile'], o=argsd['outputdir'], s=argsd['outputtostdout'])
             elif argsd['tool'] == "validate":
                 status_note('launching validator')
                 metavalidate.start(s=argsd['schema'], c=argsd['candidate'])

schema/json/dummy.json

@@ -1,4 +1,5 @@
 {
+    "access_right": "open",
     "author": [],
     "communities": [
         {
@@ -36,10 +37,11 @@
         }
     },
     "keywords": [],
-    "license": null,
+    "license": "cc-by",
     "paperLanguage": [],
     "paperSource": null,
     "publicationDate": null,
+    "publication_type": "other",
     "r_comment": [],
     "r_input": [],
     "r_output": [],
@@ -56,5 +58,6 @@
         "end": null
     },
     "title": null,
+    "upload_type": "publication",
     "version": null
 }