Merge pull request #33 from GlobalFishingWatch/31-faster-uuid

Performance optimizations

ais_tools/__init__.py

@@ -2,7 +2,7 @@
 Tools for managing AIS messages
 """
 
-__version__ = 'v0.1.4-rc.1 '
+__version__ = 'v0.1.4-rc.2 '
 __author__ = 'Paul Woods'
 __email__ = '[email protected]'
 __source__ = 'https://github.com/GlobalFishingWatch/ais-tools'

ais_tools/aivdm.py

@@ -90,7 +90,7 @@ def safe_decode(self, nmea, best_effort=False):
             msg['error'] = str(e)
         return msg
 
-    def decode(self, nmea, safe_decode_payload=False):
+    def decode(self, nmea, safe_decode_payload=False, validate_checksum=False):
         """
         Decode a single line of nmea that contains:
             a single-part AIVDM message, with or without prepended tagblock
@@ -102,7 +102,7 @@ def decode(self, nmea, safe_decode_payload=False):
 
         msg = Message(nmea)
         nmea = msg.nmea
-        parts = [expand_nmea(part) for part in split_multipart(nmea)]
+        parts = [expand_nmea(part, validate_checksum=validate_checksum) for part in split_multipart(nmea)]
         if len(parts) == 0:
             raise DecodeError('No valid AIVDM found in {}'.format(nmea))
         elif len(parts) == 1:

ais_tools/message.py

@@ -6,6 +6,7 @@
 from urllib.parse import quote as url_quote
 import posixpath as pp
 import uuid
+from hashlib import md5
 import ais_tools
 
 
@@ -43,8 +44,6 @@ def __init__(self, *args, **kwargs):
             message = args[0]
             if not message:
                 pass
-            elif isinstance(message, dict):
-                self.update(message)
             elif isinstance(message, str):
                 message = message.strip()
                 if len(message) == 0:
@@ -57,8 +56,10 @@ def __init__(self, *args, **kwargs):
                         # Nope - not JSON.  Giving up...
                         self.update(dict(nmea=message, error="JSONDecodeError: {}".format(str(e))))
                 else:
-                    # assume it's an NMEA string and pack it up in a dict
-                    self.update(dict(nmea=message))
+                    # assume it's an NMEA string
+                    self['nmea'] = message
+            elif isinstance(message, dict):
+                self.update(message)
             else:
                 raise ValueError("Unable to convert {} to NMEA message".format(message))
 
@@ -72,7 +73,11 @@ def add_source(self, source, overwrite=False):
         return self
 
     def create_uuid(self, fields=default_uuid_fields):
-        return str(UUID.create_uuid(*[str(self.get(f, '')) for f in fields]))
+        name = '|'.join((str(self.get(f, '')) for f in fields))
+        hex = md5(bytes(name, "utf-8")).digest().hex()
+        return '%s-%s-%s-%s-%s' % (
+            hex[:8], hex[8:12], hex[12:16], hex[16:20], hex[20:32]
+        )
 
     def add_uuid(self, overwrite=False, fields=default_uuid_fields):
         if self.get('uuid') is None or overwrite:
@@ -81,7 +86,7 @@ def add_uuid(self, overwrite=False, fields=default_uuid_fields):
 
     def add_parser_version(self, overwrite=False):
         if self.get('parser') is None or overwrite:
-            self['parser'] = 'ais-tools-v' + ais_tools.__version__
+            self['parser'] = 'ais-tools-' + ais_tools.__version__
         return self
 
     @classmethod

ais_tools/nmea.py

@@ -7,7 +7,7 @@
 from ais_tools.tagblock import parseTagBlock
 
 
-def expand_nmea(line):
+def expand_nmea(line, validate_checksum=False):
     try:
         tagblock, nmea = parseTagBlock(line)
     except ValueError as e:
@@ -18,7 +18,7 @@ def expand_nmea(line):
     if len(fields) < 6:
         raise DecodeError('not enough fields in nmea message')
 
-    if not isChecksumValid(nmea):
+    if validate_checksum and not isChecksumValid(nmea):
         raise DecodeError('Invalid checksum')
 
     try:

tests/test_aivdm.py

@@ -27,7 +27,6 @@ def test_decode(nmea, expected):
 
 
 @pytest.mark.parametrize("nmea,error", [
-    ('!AIVDM,1,1,,A,13`el0gP000H=3JN9jb>4?wb0>`<,1*7B', 'Invalid checksum'),
     ('!AIVDM,2,1,1,B,@,0*57', 'Expected 2 message parts to decode but found 1'),
     ('!', 'No valid AIVDM found in'),
     ('!AIVDM,1,1,,A,B99999,0*5D', 'AISTOOLS ERR: Not enough bits to decode.  Need at least 149 bits, got only 36')
@@ -38,6 +37,15 @@ def test_decode_fail(nmea, error):
         decoder.decode(nmea)
 
 
+@pytest.mark.parametrize("nmea,error", [
+    ('!AIVDM,1,1,,A,13`el0gP000H=3JN9jb>4?wb0>`<,1*7B', 'Invalid checksum'),
+])
+def test_decode_invalid_checksum(nmea, error):
+    decoder = AIVDM()
+    with pytest.raises(aivdm.libais.DecodeError, match=error):
+        decoder.decode(nmea, validate_checksum=True)
+
+
 # test for issue #1 Workaround for type 24 with bad bitcount
 def test_bad_bitcount_type_24():
     decoder = AIVDM()

tests/test_message.py

@@ -14,7 +14,7 @@ def test_uuid():
     ('!AIVDM1234567*89', {'nmea': '!AIVDM1234567*89'}),
     ({'nmea': '!AIVDM1234567*89'}, {'nmea': '!AIVDM1234567*89'}),
     ('{"nmea": "!AIVDM1234567*89"}', {'nmea': '!AIVDM1234567*89'}),
-    ('', {'nmea': ''}),
+    ('\n', {'nmea': ''}),
 ])
 def test_message_construct(msg, expected):
     assert Message(msg) == expected
@@ -53,11 +53,11 @@ def test_add_source(msg, source, overwrite, expected):
 
 
 @pytest.mark.parametrize("msg,overwrite,expected", [
-    ({}, False, 'c4a4d6e8-ea5f-5af5-8091-9afe3a73f652'),
-    ({'nmea': '!AVIDM123'}, False, 'de08b5ee-d5dc-5561-8790-4abfcbd4b953'),
+    ({}, False, '2edf2958-1665-61c5-c08c-d228e53bbcdc'),
+    ({'nmea': '!AVIDM123'}, False, '1d469a2d-5b2f-4ef9-f5ac-fb4e336e91da'),
     ({'nmea': '!AVIDM123', 'uuid': 'old'}, False, 'old'),
-    ({'nmea': '!AVIDM123', 'uuid': 'old'}, True, 'de08b5ee-d5dc-5561-8790-4abfcbd4b953'),
-    ({'nmea': '!AVIDM123', 'tagblock_timestamp': 1598653784}, True, '3c17f7c5-74fd-53af-ac9f-87861c43b217'),
+    ({'nmea': '!AVIDM123', 'uuid': 'old'}, True, '1d469a2d-5b2f-4ef9-f5ac-fb4e336e91da'),
+    ({'nmea': '!AVIDM123', 'tagblock_timestamp': 1598653784}, True, '9f2cb724-3e0b-97ff-00b5-67fcf1ca94b4'),
 ])
 def test_add_uuid(msg, overwrite, expected):
     msg = Message(msg).add_uuid(overwrite=overwrite)
@@ -104,7 +104,7 @@ def test_message_stream_add_uuid(old_uuid, add_uuid, overwrite):
     messages = [{'nmea': '!AVIDM123', 'source': 'test', 'uuid': old_uuid}]
 
     if add_uuid and (overwrite or old_uuid is None):
-        expected = '56cf351b-5e53-52b3-9f36-45a93a7d89ec'
+        expected = '123c397d-7053-8788-5984-74c73f833f37'
     else:
         expected = old_uuid
     messages = Message.stream(messages)

utils/perf-test.py

@@ -10,26 +10,52 @@
 type_1 = "!AIVDM,1,1,,A,15NTES0P00J>tC4@@FOhMgvD0D0M,0*49"
 type_18 = '!AIVDM,1,1,,A,B>cSnNP00FVur7UaC7WQ3wS1jCJJ,0*73'
 
-tests = [type_1, type_18]
+message1 = "\\s:66,c:1664582400*32\\!AIVDM,1,1,,A,15D`f63P003R@s6@@`D<Mwwp2`Rq,0*05"
+message2 = "\\g:1-2-2243,s:66,c:1664582400*47" \
+           "\\!AIVDM,2,1,1,B,5:U7dET2B4iE17KOS:0@Di0PTqE>22222222220l1@F65ut8?=lhCU3l,0*71" \
+           "\\g:2-2-2243*5A" \
+           "\\!AIVDM,2,2,1,B,p4l888888888880,2*36"
 
-for nmea in tests:
-    print(nmea)
-    print(timeit.timeit(f'AIVDM(decoder=AisToolsDecoder()).decode("{nmea}")',
-                        setup='from ais_tools.aivdm import AIVDM,LibaisDecoder,AisToolsDecoder',
-                        number=10000)
-          )
-    print()
+
+def libais_vs_aistools():
+    tests = [type_1, type_18]
+
+    for nmea in tests:
+        print(nmea)
+        print(timeit.timeit(f'AIVDM(decoder=AisToolsDecoder()).decode("{nmea}")',
+                            setup='from ais_tools.aivdm import AIVDM,LibaisDecoder,AisToolsDecoder',
+                            number=10000)
+              )
+        print()
 
 
 decoder = AIVDM(AisToolsDecoder())
 
 
-def do_something():
-    for i in range(10000):
+def decode(n):
+    for i in range(n):
         decoder.decode(type_1)
 
 
-cProfile.run('do_something()', 'perf-test.stats')
+def full_decode(n):
+    for i in range(n):
+        msg = decoder.safe_decode(message1, best_effort=True)
+        msg.add_source('source')
+        msg.add_uuid()
+        msg.add_parser_version()
+
+
+def run_perf_test(func):
+    cProfile.run(func, 'perf-test.stats')
+
+    p = pstats.Stats('perf-test.stats')
+    p.strip_dirs().sort_stats(SortKey.CUMULATIVE).print_stats(20)
+
+
+def main():
+    # run_perf_test('decode(10000)')
+    run_perf_test('full_decode(100000)')
+
 
-p = pstats.Stats('perf-test.stats')
-p.strip_dirs().sort_stats(SortKey.CUMULATIVE).print_stats(20)
+if __name__ == "__main__":
+    main()