Merge pull request #58 from GlobalFishingWatch/more-talkers

More talkers
GlobalFishingWatch · Apr 4, 2024 · 95b08e6 · 95b08e6
2 parents 8a98844 + 625eb87
commit 95b08e6
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 3 deletions.
diff --git a/ais_tools/normalize.py b/ais_tools/normalize.py
@@ -5,7 +5,7 @@
 from enum import Enum
 
 
-REGEX_NMEA = re.compile(r'!(?:AI|BS|AB)VDM[^*]+\*[0-9A-Fa-f]{2}')
+REGEX_NMEA = re.compile(r'!(?:AB|AI|AN|BS)VD[MO][^*]+\*[0-9A-Fa-f]{2}')
 SKIP_MESSAGE_IF_FIELD_PRESENT = ['error']
 SKIP_MESSAGE_IF_FIELD_ABSENT = ['id', 'mmsi', 'tagblock_timestamp']
 AIS_TYPES = frozenset([1, 2, 3, 4, 5, 9, 11, 17, 18, 19, 21, 24, 27])

diff --git a/tests/test_normalize.py b/tests/test_normalize.py
@@ -188,11 +188,22 @@ def test_normalize_draught(message, expected):
 
 
 @pytest.mark.parametrize("value, expected", [
+    ('invalid', []),
+    ('!AIVDM*00', []),
+    ('!AIVDM_*00', ['!AIVDM_*00']),
+    ('!AIVDM_*0', []),
+    ('!XXVDM_*00', []),
+    ('!AIVDX_*00', []),
+    ('!AIVDO_*00', ['!AIVDO_*00']),
+    ('!ABVDM_*00', ['!ABVDM_*00']),
+    ('!ABVDO_*00', ['!ABVDO_*00']),
+    ('!ANVDM_*00', ['!ANVDM_*00']),
+    ('!ANVDO_*00', ['!ANVDO_*00']),
+    ('!BSVDM_*00', ['!BSVDM_*00']),
+    ('!BSVDO_*00', ['!BSVDO_*00']),
     ('!AIVDM,2,2,2,A,@,0*57', ['!AIVDM,2,2,2,A,@,0*57']),
     ('Not-included!AIVDM,2,2,2,A,@,0*57Also-not-included', ['!AIVDM,2,2,2,A,@,0*57']),
     ('!AIVDM,2,2,2,A,@,0*57not-included!AIVDM,2,2,2,A,@,0*57', ['!AIVDM,2,2,2,A,@,0*57', '!AIVDM,2,2,2,A,@,0*57']),
-    ('!BSVDM,2,2,2,A,@,0*57', ['!BSVDM,2,2,2,A,@,0*57']),
-    ('!ABVDM,2,2,2,A,@,0*57', ['!ABVDM,2,2,2,A,@,0*57']),
 ])
 def test_nmea_regex(value, expected):
     assert re.findall(REGEX_NMEA, value) == expected
@@ -206,6 +217,7 @@ def test_nmea_regex(value, expected):
     ({'nmea': 'invalid', 'tagblock_timestamp': 1707443048}, None),
     ({"nmea": "!AIVDM,1,1,,A,H69@rrS3S?SR3G2D000000000000,0*2e",
       "tagblock_timestamp": 1712156268}, '06f1f1b00815aa10'),
+    ({'nmea': '!AIVDM', 'tagblock_timestamp': 0}, None),
 ])
 def test_normalize_dedup_key(message, expected):
     assert normalize_dedup_key(message) == expected