upd broker

Author: 7048730

.gitignore

@@ -1,6 +1,8 @@
 # specific
 extract/tests/all
+broker/tests/all
 metadata_raw.json
+metadata_o2r.json
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.travis.yml

@@ -19,3 +19,4 @@ install:
   - pip install -r requirements.txt
 script:
   - python o2rmeta.py -debug extract -i extract/tests -o extract/tests -xo
+  - python o2rmeta.py -debug validate -s schema/json/o2r-meta-schema.json -c schema/json/example1-valid.json

README.md

@@ -61,11 +61,11 @@ Each tool then has a number of required arguments:
 
 #(1) Extractor tool:
 
-	python o2rmeta extract -i <INPUT_DIR> -s|-o <OUTPUT_DIR> [-xo] [-m] [-xml] [-ercid <ERC_ID>]
+	python o2rmeta.py extract -i <INPUT_DIR> -s|-o <OUTPUT_DIR> [-xo] [-m] [-xml] [-ercid <ERC_ID>]
 
 Example call:
 
-	python o2rmeta extract -i extract/tests -o extract/tests -xo
+	python o2rmeta.py -debug extract -i extract/tests -o extract/tests -xo
 
 Explanation of the switches:
 
@@ -79,15 +79,29 @@ Explanation of the switches:
 
 
 #(2) broker
-TDB
+
+    python o2rmeta.py broker -i <INPUT_DIR/FILE> -m <MAPPING_FILE> -s|-o <OUTPUT_DIR>
+	
+Example call:
+	
+    python o2rmeta.py -debug broker -i broker/tests -m broker/mappings/o2r-map.json -o broker/tests/all
+
+Explanation of the switches:
+
++ `-i` <INPUT_DIR> : required starting path for recursive search for parsable files. 
++ `-m` <MAPPING_FILE> : required path to a json mapping file that holds translation instructions for the metadata mappings. #TBD: mapping json schema.
++ `-s`: option to print out results to console. This switch is mutually exclusive with `-o`. At least one of them must be given
++ `-o` <OUTPUT_DIR> : required output path, where data should be saved. If the directory does not exist, it will be created on runtime. This switch is mutually exclusive with `-s`. At least one of them must be given.
+
+
 
 #(3) Validator tool:
 
-	python o2rmeta validate -s <SCHEMA> -c <CANDIDATE>
+	python o2rmeta.py validate -s <SCHEMA> -c <CANDIDATE>
 
 Example call:
 
-	python o2rmeta validate -s schema/json/o2r-meta-schema.json -c schema/json/example1-valid.json
+	python o2rmeta.py -debug validate -s schema/json/o2r-meta-schema.json -c schema/json/example1-valid.json
 
 Explanation of the switches:
 

broker/Dockerfile

@@ -1,6 +1,7 @@
 {
 	"Settings": {
 		"name": "o2r",
+		"outputfile": "metadata_o2r",
 		"map_description": "maps raw extracted metadata to o2r schema compliant metadata",
 		"mode": "json",
 		"root": ""

broker/mappings/o2r-map.json

@@ -34,7 +34,7 @@ def do_outputs(output_data, out_mode, out_name, file_ext):
     else:
         try:
             # output path is given in <out_mode>
-            output_filename = os.path.join(out_mode, '_'.join((out_name, file_ext)))
+            output_filename = os.path.join(out_mode, ''.join((out_name, file_ext)))
             if not os.path.exists(out_mode):
                 os.makedirs(out_mode)
             with open(output_filename, 'w', encoding='utf-8') as outfile:
@@ -191,87 +191,69 @@ def map_xml(element, value, map_data, xml_root):
 
 
 def status_note(msg):
-    print(''.join(('[metabroker] ', msg)))
+    print(''.join(('[o2rmeta][broker] ', msg)))
 
 
 # Main
-if __name__ == "__main__":
-    if sys.version_info[0] < 3:
-        # py2
-        status_note('requires py3k or later')
-        sys.exit()
+def start(**kwargs):
+    input_dir = kwargs.get('i', None)
+    output_dir = kwargs.get('o', None)
+    output_to_console = kwargs.get('s', None)
+    seperator = '#' #<-- make this generic
+    my_map = kwargs.get('m', None)
+    # output mode
+    if output_to_console:
+        output_mode = '@s'
+    elif output_dir:
+        output_mode = output_dir
+        if not os.path.isdir(output_dir):
+            status_note(''.join(('directory at <', output_dir, '> will be created during extraction...')))
     else:
-        my_version = 1
-        my_mod = ''
-        try:
-            my_mod = datetime.datetime.fromtimestamp(os.stat(__file__).st_mtime)
-        except OSError:
-            pass
-        status_note(''.join(('v', str(my_version), ' - ', str(my_mod))))
-        parser = argparse.ArgumentParser(description='description')
-        parser.add_argument('-m', '--map', help='name of the mapping file', required=True)
-        parser.add_argument('-i', '--inputdir', help='input directory', required=True)
-        group = parser.add_mutually_exclusive_group(required=True)
-        group.add_argument('-o', '--outputdir', help='output directory for extraction docs')
-        group.add_argument('-s', '--outputtostdout', help='output the result of the extraction to stdout', action='store_true', default=False)
-        args = parser.parse_args()
-        args_dict = vars(args)
-        input_dir = args_dict['inputdir']
-        output_dir = args_dict['outputdir']
-        output_to_console = args_dict['outputtostdout']
-        seperator = '#' #<-- make this generic
-        my_map = args_dict['map']
-        # output mode
-        if output_to_console:
-            output_mode = '@s'
-        elif output_dir:
-            output_mode = output_dir
-            if not os.path.isdir(output_dir):
-                status_note(''.join(('directory <', output_dir, '> will be created during extraction...')))
-        else:
-            # not possible currently because output arg group is on mutual exclusive
-            output_mode = '@none'
+        # not possible currently because output arg group is on mutual exclusive
+        output_mode = '@none'
 
-        # open map file and find out mode
-        try:
-            with open(os.path.join('mappings', my_map), encoding='utf-8') as data_file:
-                map_file = json.load(data_file)
-                settings_data = map_file['Settings']
-                map_data = map_file['Map']
-                my_mode = settings_data['mode']
-        except:
-            raise
-        # distinguish format for output
-        if my_mode == 'json':
-            for file in os.listdir(input_dir):
-                if os.path.basename(file).startswith('meta_'):
-                    json_output = {}
-                    with open(os.path.join(input_dir, file), encoding='utf-8') as data_file:
-                        test_data = json.load(data_file)
-                    for element in test_data:
-                        try:
-                            map_json(element, test_data[element], map_data, json_output)
-                        except:
-                            raise
-                    do_outputs(json_output, output_mode, 'o2r_'+os.path.splitext(file)[0], '.json')
-        elif my_mode == 'txt':
-            # to do: handle txt based maps like bagit
-            txt_output = ''
-            do_outputs(txt_output, output_mode, '.txt')
-        elif my_mode == 'xml':
-            root = ET.Element(settings_data['root'])
-            # to do: generify for complex xml maps
-            root.set('xmlns', settings_data['root@xmlns'])
-            root.set('xmlns:xsi', settings_data['root@xmlns:xsi'])
-            root.set('xsi:schemaLocation', settings_data['root@xsi:schemaLocation'])
-            with open(os.path.join('tests', 'meta_test1.json'), encoding='utf-8') as data_file:
-                test_data = json.load(data_file)
-            for element in test_data:
-                try:
-                    map_xml(element, test_data[element], map_data, root)
-                except:
-                    raise
-            output = ET.tostring(root, encoding='utf8', method='xml')
-            do_outputs(minidom.parseString(output).toprettyxml(indent='\t'), output_mode, '.xml')
-        else:
-            print('[metabroker] ! error: cannot process map mode of <' + my_map + '>')
+    # open map file and find out mode
+    try:
+        with open(my_map, encoding='utf-8') as data_file:
+            map_file = json.load(data_file)
+            settings_data = map_file['Settings']
+            map_data = map_file['Map']
+            my_mode = settings_data['mode']
+    except:
+        raise
+    # distinguish format for output
+    if my_mode == 'json':
+        # try parse all possible metadata files:
+        for file in os.listdir(input_dir):
+            if os.path.basename(file).startswith('metadata_'):
+                json_output = {}
+                with open(os.path.join(input_dir, file), encoding='utf-8') as data_file:
+                    test_data = json.load(data_file)
+                for element in test_data:
+                    try:
+                        map_json(element, test_data[element], map_data, json_output)
+                    except:
+                        raise
+                ##do_outputs(json_output, output_mode, 'o2r_'+os.path.splitext(file)[0], '.json')
+                do_outputs(json_output, output_mode, settings_data['outputfile'], '.json')
+    elif my_mode == 'txt':
+        # to do: handle txt based maps like bagit
+        txt_output = ''
+        do_outputs(txt_output, output_mode, '.txt')
+    elif my_mode == 'xml':
+        root = ET.Element(settings_data['root'])
+        # to do: generify for complex xml maps
+        root.set('xmlns', settings_data['root@xmlns'])
+        root.set('xmlns:xsi', settings_data['root@xmlns:xsi'])
+        root.set('xsi:schemaLocation', settings_data['root@xsi:schemaLocation'])
+        with open(os.path.join('tests', 'meta_test1.json'), encoding='utf-8') as data_file:
+            test_data = json.load(data_file)
+        for element in test_data:
+            try:
+                map_xml(element, test_data[element], map_data, root)
+            except:
+                raise
+        output = ET.tostring(root, encoding='utf8', method='xml')
+        do_outputs(minidom.parseString(output).toprettyxml(indent='\t'), output_mode, '.xml')
+    else:
+        status_note('! error: cannot process map mode of <' + my_map + '>')

broker/metabroker.py

@@ -342,18 +342,14 @@ def start(**kwargs):
     rule_set_rmd_multiline = ['\t'.join(('yaml', r'---\n(.*?)\n---\n')),
                               '\t'.join(('rblock', r'\`{3}(.*)\`{3}'))]
     # other parameters
-    ##global packlist_geosci
-    ##packlist_geosci = 'extract/list_geosci.txt'
-    ##global packlist_crantop100
-    ##packlist_crantop100 = 'extract/list_crantop100.txt'
     nr = 0  # number of files processed
     if skip_orcid:
         status_note('orcid api search disabled...')
     global md_paper_source
     md_paper_source = ''
     # md_bbox_list = {}
     global MASTER_MD_DICT
-    MASTER_MD_DICT = {}  # todo: this one is being updated per function call
+    MASTER_MD_DICT = {}  # this one is being updated per function call
     bagit_txt_file = None
     global compare_extracted
     compare_extracted = {}  # dict for evaluations to find best metafile for main output

broker/requirements.txt

@@ -39,8 +39,8 @@ def status_note(msg, **kwargs):
 
 # Main
 if __name__ == "__main__":
-    if sys.version_info[0] < 3 and sys.version_info[1] < 6:  # target py36
-        status_note('requires python 3.6')
+    if sys.version_info[0] < 3 and sys.version_info[1] < 4:  # target py36
+        status_note('requires python 3.4')
         sys.exit(0)
     else:
         # arg parse setup:
@@ -61,6 +61,14 @@ def status_note(msg, **kwargs):
         extractor.add_argument('-m', '--metafiles', help='output all metafiles', action='store_true', default=False,
                             required=False)
         # - - - - - - - - - - - - - - - - - -
+        broker = subparsers.add_parser("broker")
+        broker.add_argument("-i", "--inputdir", type=str, required=True)
+        broker.add_argument("-m", "--map", type=str, required=True)
+        group = broker.add_mutually_exclusive_group(required=True)
+        group.add_argument('-o', '--outputdir', help='output directory for brokering docs')
+        group.add_argument('-s', '--outputtostdout', help='output the result of the brokering to stdout',
+                           action='store_true', default=False)
+        # - - - - - - - - - - - - - - - - - -
         validator = subparsers.add_parser("validate")
         validator.add_argument("-s", "--schema", type=str, required=True)
         validator.add_argument("-c", "--candidate", type=str, required=True)
@@ -74,7 +82,7 @@ def status_note(msg, **kwargs):
         except OSError:
             pass
         status_note(''.join(('v', str(my_version), ' - ', str(my_mod))), debug=argsd['debug'])
-        status_note(''.join(('running under python ', str(sys.version_info[0]), '.' , str(sys.version_info[1]), '.', str(
+        status_note(''.join(('running under python ', str(sys.version_info[0]), '.', str(sys.version_info[1]), '.', str(
             sys.version_info[2]))), debug=argsd['debug'])
         status_note(''.join(('received arguments: ', str(argsd))), debug=argsd['debug'])
         try:
@@ -83,7 +91,7 @@ def status_note(msg, **kwargs):
                 metaextract.start(i=argsd['inputdir'], o=argsd['outputdir'], s=argsd['outputtostdout'], xo=argsd['skiporcid'], e=argsd['ercid'], m=argsd['metafiles'], xml=argsd['modexml'])
             elif argsd['tool'] == "broker":
                 status_note('launching broker')
-                print('TBD')  # todo
+                metabroker.start(i=argsd['inputdir'], o=argsd['outputdir'], s=argsd['outputtostdout'], m=argsd['map'])
             elif argsd['tool'] == "validate":
                 status_note('launching validator')
                 metavalidate.start(s=argsd['schema'], c=argsd['candidate'])

broker/tests/meta_test1.json broker/tests/metadata_raw_1.json

@@ -75,7 +75,7 @@ def xml_validate(c, s, bln_c_http, bln_s_http):
 
 
 def status_note(msg):
-    print(''.join(('[validate] ', msg)))
+    print(''.join(('[o2rmeta][validate] ', msg)))
 
 
 # main