add normalization for bouy shipnames

pipe_segment/transform/normalize.py

@@ -3,7 +3,7 @@
 from stdnum import imo as imo_validator
 
 from shipdataprocess.normalize import normalize_callsign
-from shipdataprocess.normalize import normalize_shipname
+from shipdataprocess.normalize import normalize_shipname_parts
 
 class NormalizeDoFn(beam.DoFn):
     @staticmethod
@@ -13,8 +13,13 @@ def add_normalized_field(msg, fn, field):
             msg['n_'+field] = normalized
 
     def process(self, msg):
-        self.add_normalized_field(msg, normalize_shipname, 'shipname')
-        self.add_normalized_field(msg, normalize_callsign, 'callsign')
+        n_shipname = normalize_shipname_parts(msg.get('shipname'))['basename']
+        if n_shipname:
+            msg['n_shipname'] = n_shipname
+
+        n_callsign = normalize_callsign(msg.get('callsign'))
+        if n_callsign:
+            msg['n_callsign'] = n_callsign
 
         imo = msg.get('imo')
         if imo_validator.is_valid(str(imo)):

requirements.txt

@@ -1,3 +1,3 @@
 https://codeload.github.com/GlobalFishingWatch/pipe-tools/tar.gz/v0.1.7#egg=pipe-tools-0.1.7
 https://codeload.github.com/Skytruth/gpsdio-segment/tar.gz/v0.13-dev#egg=gpsdio-segment-0.13-dev0
-https://codeload.github.com/GlobalFishingWatch/ShipDataProcess/tar.gz/70f52826074b8b65a2811ea3ae626a9373cdb83d#egg=shipdataprocess-0.5.0
+https://codeload.github.com/GlobalFishingWatch/ShipDataProcess/tar.gz/06b8495f55c6a44306ca9865b4c07d51d7ad5a7b#egg=shipdataprocess-0.5.8

setup.py

@@ -23,7 +23,7 @@
     "python-stdnum",
     "pipe-tools==0.1.7",
     "gpsdio-segment==0.13-dev0",
-    "shipdataprocess==0.5.0",
+    "shipdataprocess==0.5.8",
     "jinja2-cli",
 ]
 

tests/test_transforms.py

@@ -153,6 +153,7 @@ def test_segment_out_in(self, temp_dir):
     @pytest.mark.parametrize("message, expected", [
         ({}, {}),
         ({'shipname': 'f/v boaty Mc Boatface'}, {'n_shipname': 'BOATYMCBOATFACE'}),
+        ({'shipname': 'Bouy 42%'}, {'n_shipname': 'BOUY'}),
         ({'callsign': '@@123'}, {'n_callsign': '123'}),
         ({'imo': 8814275}, {'n_imo': 8814275}),
     ])