From 45aca563503142a9d4b7e60f3833c0e1eb838fc2 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Fri, 24 May 2024 13:44:55 -0400 Subject: [PATCH 01/16] Performance improvement in normalize_dedup_key() --- ais_tools/normalize.py | 8 +++----- pyproject.toml | 1 + tests/test_normalize.py | 10 +++++----- utils/perf-test.py | 11 ++++++++++- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ais_tools/normalize.py b/ais_tools/normalize.py index 139acfc..cf3632a 100644 --- a/ais_tools/normalize.py +++ b/ais_tools/normalize.py @@ -1,6 +1,6 @@ from typing import Optional, Any from datetime import datetime -import hashlib +import xxhash import re from enum import Enum @@ -168,16 +168,14 @@ def normalize_dedup_key(message: dict) -> Optional[str]: if 'nmea' not in message or 'tagblock_timestamp' not in message: return None - nmea = ''.join(re.findall(REGEX_NMEA, message['nmea'])) + nmea = ''.join(REGEX_NMEA.findall(message['nmea'])) if not nmea: return None # no nmea found in message timestamp = int(message['tagblock_timestamp'] / 60) key = f'{nmea}_{timestamp}'.encode('utf-8') - h = hashlib.sha1() - h.update(key) - return h.hexdigest()[:16] + return xxhash.xxh3_64_hexdigest(key) def map_field(message: dict, source_field: str = None) -> Optional[Any]: diff --git a/pyproject.toml b/pyproject.toml index bf060ae..bd1931c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "bitarray", "cbitstruct", "udatetime", + "xxhash" ] [project.urls] diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 27fa913..abf2470 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -229,12 +229,12 @@ def test_nmea_regex(value, expected): @pytest.mark.parametrize("message,expected", [ ({'tagblock_timestamp': 1707443048}, None), - ({'nmea': '!AIVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, '745f4bde2318c974'), - ({'nmea': '!BSVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, 'a6926b3f62eeb7d7'), - ({'nmea': '!BSVDM,2,2,2,B,@,0*57', 'tagblock_timestamp': 1707443048}, 'd3972916d1a17048'), + ({'nmea': '!AIVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, '784dcbf153c04531'), + ({'nmea': '!BSVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, 'da0adb5959aec091'), + ({'nmea': '!BSVDM,2,2,2,B,@,0*57', 'tagblock_timestamp': 1707443048}, '70bbc54a4bfc5daa'), ({'nmea': 'invalid', 'tagblock_timestamp': 1707443048}, None), ({"nmea": "!AIVDM,1,1,,A,H69@rrS3S?SR3G2D000000000000,0*2e", - "tagblock_timestamp": 1712156268}, '06f1f1b00815aa10'), + "tagblock_timestamp": 1712156268}, 'e804595091021cd8'), ({'nmea': '!AIVDM', 'tagblock_timestamp': 0}, None), ]) def test_normalize_dedup_key(message, expected): @@ -287,7 +287,7 @@ def test_filter_message(message, expected): ({'dim_c': 1, 'dim_d': 1}, {'width': 2}), ({'type_and_cargo': 30}, {'shiptype': 'Fishing'}), ({'nmea': '!AIVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, - {'timestamp': '2024-02-09T01:44:08Z', 'dedup_key': '745f4bde2318c974'}), + {'timestamp': '2024-02-09T01:44:08Z', 'dedup_key': '784dcbf153c04531'}), ({'year': 2024, 'month': 4, 'day': 3, 'hour': 2, 'minute': 1, 'second': 0}, {'tx_timestamp': '2024-04-03T02:01:00Z'}) ]) diff --git a/utils/perf-test.py b/utils/perf-test.py index d707bcc..c7f7d0b 100644 --- a/utils/perf-test.py +++ b/utils/perf-test.py @@ -3,6 +3,7 @@ import pstats from pstats import SortKey from ais_tools.aivdm import AIVDM, AisToolsDecoder +from ais_tools.normalize import normalize_dedup_key # from ais_tools.aivdm import LibaisDecoder @@ -60,6 +61,14 @@ def full_decode(n): msg.add_parser_version() +def test_normalize_dedup_key(n): + msg = { + 'tagblock_timestamp': 123456789, + 'nmea': "\\!AIVDM,2,1,1,B,5:U7dET2B4iE17KOS:0@Di0PTqE>22222222220l1@F65ut8?=lhCU3l,0*71" + } + for i in range (n): + _ = normalize_dedup_key(msg) + def run_perf_test(func): cProfile.run(func, 'perf-test.stats') @@ -68,7 +77,7 @@ def run_perf_test(func): def main(): - run_perf_test('decode(10000)') + run_perf_test('test_normalize_dedup_key(1000000)') # run_perf_test('full_decode(100000)') # checksum_compare() From 83312f0504f9d209482f8a7ef02920a92ffbf095 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Mon, 27 May 2024 20:11:52 -0400 Subject: [PATCH 02/16] catch edge case where day value in tx_timestamp is invalid for the given month --- ais_tools/normalize.py | 7 +++++-- tests/test_normalize.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ais_tools/normalize.py b/ais_tools/normalize.py index cf3632a..7768075 100644 --- a/ais_tools/normalize.py +++ b/ais_tools/normalize.py @@ -36,7 +36,7 @@ def in_range(value, valid_range): fields = { 'year': (1, 9999), 'month': (1, 12), - 'day': (1,31), + 'day': (1, 31), 'hour': (0, 23), 'minute': (0, 59), 'second': (0, 59) @@ -44,7 +44,10 @@ def in_range(value, valid_range): values = [(f, message.get(f), valid_range) for f, valid_range in fields.items()] if all(in_range(value, valid_range) for _, value, valid_range in values): values = {f: value for f, value, _ in values} - return datetime(**values).isoformat(timespec='seconds') + 'Z' + try: + return datetime(**values).isoformat(timespec='seconds') + 'Z' + except ValueError: + return None def coord_type(val: float, _min: float, _max: float, unavailable: float) -> POSITION_TYPE: diff --git a/tests/test_normalize.py b/tests/test_normalize.py index abf2470..2938c7c 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -289,7 +289,9 @@ def test_filter_message(message, expected): ({'nmea': '!AIVDM,2,2,2,A,@,0*57', 'tagblock_timestamp': 1707443048}, {'timestamp': '2024-02-09T01:44:08Z', 'dedup_key': '784dcbf153c04531'}), ({'year': 2024, 'month': 4, 'day': 3, 'hour': 2, 'minute': 1, 'second': 0}, - {'tx_timestamp': '2024-04-03T02:01:00Z'}) + {'tx_timestamp': '2024-04-03T02:01:00Z'}), + ({'year': 2024, 'month': 2, 'day': 31, 'hour': 0, 'minute': 0, 'second': 0}, + {}), ]) def test_normalize_message(message, expected): assert normalize_message(message, DEFAULT_FIELD_TRANSFORMS) == expected From 849e21a8be73205c03fd30867d1828edf132089a Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Mon, 27 May 2024 20:19:15 -0400 Subject: [PATCH 03/16] update dev version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bd1931c..01fbd11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7" +version = "v0.1.7.dev1" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"}, From 97933a70d173db91f924230e4d821fdc57e47a9d Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Fri, 31 May 2024 08:22:22 -0400 Subject: [PATCH 04/16] fix memory leak in checksum --- ais_tools/core/methods.c | 44 +++++++++++++++++++++++++++++++--------- utils/mem-test.py | 21 +++++++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 utils/mem-test.py diff --git a/ais_tools/core/methods.c b/ais_tools/core/methods.c index 873def8..6a96742 100644 --- a/ais_tools/core/methods.c +++ b/ais_tools/core/methods.c @@ -8,45 +8,69 @@ PyObject * method_compute_checksum(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { + PyObject *py_str; + PyObject *result; const char *str; if (nargs != 1) return PyErr_Format(PyExc_TypeError, "compute_checksum expects 1 argument"); - str = PyUnicode_AsUTF8(PyObject_Str(args[0])); - return PyLong_FromLong(checksum(str)); + py_str = PyObject_Str(args[0]); + + str = PyUnicode_AsUTF8(py_str); + + result = PyLong_FromLong(checksum(str)); + + Py_DECREF(py_str); + + return result; } PyObject * method_compute_checksum_str(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { + PyObject *py_str; + PyObject *result; const char *str; char c_str[3]; if (nargs != 1) return PyErr_Format(PyExc_TypeError, "checksum_str expects 1 argument"); - str = PyUnicode_AsUTF8(PyObject_Str(args[0])); + py_str = PyObject_Str(args[0]); + + str = PyUnicode_AsUTF8(py_str); + checksum_str(c_str, str, ARRAY_LENGTH(c_str)); - return PyUnicode_FromString(c_str); + + result = PyUnicode_FromString(c_str); + + Py_DECREF(py_str); + + return result; } PyObject * method_is_checksum_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { + PyObject *py_str; + PyObject *result; const char *str; char buffer[MAX_SENTENCE_LENGTH]; if (nargs != 1) return PyErr_Format(PyExc_TypeError, "checksum_str expects 1 argument"); - str = PyUnicode_AsUTF8(PyObject_Str(args[0])); + py_str = PyObject_Str(args[0]); - if (safe_strcpy(buffer, str, ARRAY_LENGTH(buffer)) >= ARRAY_LENGTH(buffer)) - return PyErr_Format(PyExc_ValueError, "String too long"); + str = PyUnicode_AsUTF8(py_str); - if (is_checksum_valid(buffer)) - Py_RETURN_TRUE; + if (safe_strcpy(buffer, str, ARRAY_LENGTH(buffer)) >= ARRAY_LENGTH(buffer)) + result = PyErr_Format(PyExc_ValueError, "String too long"); else - Py_RETURN_FALSE; + result = is_checksum_valid(buffer) ? Py_NewRef(Py_True) : Py_NewRef(Py_False); + + Py_DECREF(py_str); + + return result; } diff --git a/utils/mem-test.py b/utils/mem-test.py new file mode 100644 index 0000000..743d93a --- /dev/null +++ b/utils/mem-test.py @@ -0,0 +1,21 @@ + +from memory_profiler import profile + +from ais_tools.core import checksum +from ais_tools.core import checksum_str +from ais_tools.core import is_checksum_valid + +def run_checksum(n): + for i in range(n): + str = ''.join(['a','b']) + c = checksum(str) + c = checksum_str(str) + c = is_checksum_valid(str) + + +@profile +def run_test(): + run_checksum(1000000) + + +run_test() \ No newline at end of file From 6acba16446257a7d31a44bc596c854c3863b27f0 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Fri, 31 May 2024 08:41:01 -0400 Subject: [PATCH 05/16] make it compile for <= python 3.9 --- ais_tools/core/methods.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ais_tools/core/methods.c b/ais_tools/core/methods.c index 6a96742..9730a5b 100644 --- a/ais_tools/core/methods.c +++ b/ais_tools/core/methods.c @@ -54,9 +54,9 @@ PyObject * method_is_checksum_valid(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *py_str; - PyObject *result; const char *str; char buffer[MAX_SENTENCE_LENGTH]; + size_t len; if (nargs != 1) return PyErr_Format(PyExc_TypeError, "checksum_str expects 1 argument"); @@ -65,12 +65,14 @@ method_is_checksum_valid(PyObject *module, PyObject *const *args, Py_ssize_t nar str = PyUnicode_AsUTF8(py_str); - if (safe_strcpy(buffer, str, ARRAY_LENGTH(buffer)) >= ARRAY_LENGTH(buffer)) - result = PyErr_Format(PyExc_ValueError, "String too long"); - else - result = is_checksum_valid(buffer) ? Py_NewRef(Py_True) : Py_NewRef(Py_False); + len = safe_strcpy(buffer, str, ARRAY_LENGTH(buffer)); Py_DECREF(py_str); - return result; + if (len >= ARRAY_LENGTH(buffer)) + return PyErr_Format(PyExc_ValueError, "String too long"); + else if (is_checksum_valid(buffer)) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; } From 643b2b514a5cad3e579f549109ce633ea85a06b3 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Fri, 31 May 2024 10:00:57 -0400 Subject: [PATCH 06/16] bump dev version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01fbd11..8ed5fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7.dev1" +version = "v0.1.7.dev2" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"}, From 1e7e530e2c9c744e35abf1e369b0fc47588d01f8 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Sun, 2 Jun 2024 20:05:22 -0400 Subject: [PATCH 07/16] Remove unneded dependency on deprecated cdefs.h in core --- ais_tools/core/strcpy.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ais_tools/core/strcpy.c b/ais_tools/core/strcpy.c index a1645b1..df1f9c4 100644 --- a/ais_tools/core/strcpy.c +++ b/ais_tools/core/strcpy.c @@ -1,4 +1,3 @@ -#include #include /* From 96c6dcf56b319c05a00001370d45a6b6d1c9e4a1 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Sun, 2 Jun 2024 20:11:04 -0400 Subject: [PATCH 08/16] bump dev version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8ed5fdc..4df1065 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7.dev2" +version = "v0.1.7.dev3" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"}, From 5884afc76716caf3dd0732c2d79f7f8f8b8a74b1 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Thu, 6 Jun 2024 20:20:22 -0400 Subject: [PATCH 09/16] implement best effort decoding for type 1,2,3 when there are extra bits present --- ais_tools/ais.py | 4 ++++ ais_tools/ais_1_2_3.py | 10 ++++++++++ sample/type1-bad-bitcount.nmea | 13 +++++++++++++ tests/test_aivdm.py | 4 +++- 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 ais_tools/ais_1_2_3.py create mode 100644 sample/type1-bad-bitcount.nmea diff --git a/ais_tools/ais.py b/ais_tools/ais.py index 7cd5853..6058b5f 100644 --- a/ais_tools/ais.py +++ b/ais_tools/ais.py @@ -1,5 +1,6 @@ from ais_tools.transcode import DecodeError from ais_tools.transcode import ASCII8toAIS6 +from ais_tools import ais_1_2_3 from ais_tools import ais5 from ais_tools import ais8 from ais_tools import ais9 @@ -19,6 +20,9 @@ } decode_fn = { + 1: ais_1_2_3.ais_1_2_3_decode, + 2: ais_1_2_3.ais_1_2_3_decode, + 3: ais_1_2_3.ais_1_2_3_decode, 5: ais5.ais5_decode, 8: ais8.ais8_decode, 9: ais9.ais9_decode, diff --git a/ais_tools/ais_1_2_3.py b/ais_tools/ais_1_2_3.py new file mode 100644 index 0000000..510d02e --- /dev/null +++ b/ais_tools/ais_1_2_3.py @@ -0,0 +1,10 @@ +import ais as libais +from ais import DecodeError + + +def ais_1_2_3_decode(body, pad): + try: + return libais.decode(body[:28], 0) + except DecodeError as e: + raise DecodeError(f'TYPE 1-2-3 LIBAIS ERR: {str(e)}') + diff --git a/sample/type1-bad-bitcount.nmea b/sample/type1-bad-bitcount.nmea new file mode 100644 index 0000000..1e8287b --- /dev/null +++ b/sample/type1-bad-bitcount.nmea @@ -0,0 +1,13 @@ +\s:rMT4097,t:ais-listener,c:1697012399*7E\!AIVDM,1,1,,A,13UuUj0P00QgDCLEiGAcOOwl2<0Igw;,0*28 +\s:rMT1200,t:ais-listener,c:1697012048*78\!AIVDM,1,1,,A,33lP4T5P00QJoU6Fd12D`Jal<,0*7F +\s:rMT1200,t:ais-listener,c:1697011998*7F\!AIVDM,1,1,,A,13R3iq00001LhHDFVNd@RC@R0<0FPF=,0*6C +\s:rMT6030,t:ais-listener,c:1697011876*78\!AIVDM,1,1,,B,15Ca7j002HQer4PClUD<1qbH089PRn4,0*07 +\s:rMT6168,t:ais-listener,c:1697014351*7F\!AIVDM,1,1,,B,16eg3D1P00RT5kHCfSje>Ovr:`AhPc=,0*57 +\s:rMT6030,t:ais-listener,c:1697014102*77\!AIVDM,1,1,,A,13R3=u3P00QfIwhCt;09V?v`0530Li7,0*61 +\s:rMT1200,t:ais-listener,c:1697011340*70\!AIVDM,1,1,,B,13VOUT002W1KM1hFbSbcrITb0<1Muw>,0*03 +\s:rMT6030,t:ais-listener,c:1717190215*76\!AIVDM,1,1,,A,144d5;0uiHQjSVjC6w:iL1Ab26a05d4,0*72 +\s:rMT6030,t:ais-listener,c:1717190199*71\!AIVDM,1,1,,A,13lHkf001C1o?`HC3=Oc@8w408Cos:<,0*3D +\s:rMT6030,t:ais-listener,c:1717190200*72\!AIVDM,1,1,,B,19@9=p0P1<1mUP`C7ttkBOw800RqWe=,0*3C +\s:rMT1200,t:ais-listener,c:1717190111*77\!AIVDM,1,1,,B,34i:=V50001K=nbFbsWeH42D2000j0<,0*5F +\s:rMT1200,t:ais-listener,c:1717189944*77\!AIVDM,1,1,,B,33TVdn1P01QK>R`Fc78v4?vh2DlbI03,0*20 + diff --git a/tests/test_aivdm.py b/tests/test_aivdm.py index 4de0da2..83a2433 100644 --- a/tests/test_aivdm.py +++ b/tests/test_aivdm.py @@ -19,7 +19,9 @@ ('\\s:66,c:1663246931*35\\!AIVDM,1,1,,,9001?BP=h:qJ9vb;:f7EN1h240Rb,0*3F', {'alt_sensor': 0, 'assigned_mode': False}), ('\\c:1712130370,s:dynamic,t:spire*55\\!AIVDM,1,1,,A,403wboivQ1WfE`4gnt5MJT?024rp,0*24', - {'year': 2024, 'month': 4, 'day': 3, 'hour': 7, 'minute': 46, 'second': 21}) + {'year': 2024, 'month': 4, 'day': 3, 'hour': 7, 'minute': 46, 'second': 21}), + ('\\s:rMT4097,t:ais-listener,c:1697012399*7E\\!AIVDM,1,1,,A,13UuUj0P00QgDCLEiGAcOOwl2<0Igw;,0*28', + {'id': 1, 'mmsi': 241133000}), ]) def test_decode(nmea, expected): decoder = AIVDM() From fc77d3a44589b7cfc748bb083250bf58efc90d8c Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Thu, 6 Jun 2024 20:32:48 -0400 Subject: [PATCH 10/16] add test for failure case --- ais_tools/ais_1_2_3.py | 6 +----- tests/test_aivdm.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/ais_tools/ais_1_2_3.py b/ais_tools/ais_1_2_3.py index 510d02e..d50d8e2 100644 --- a/ais_tools/ais_1_2_3.py +++ b/ais_tools/ais_1_2_3.py @@ -3,8 +3,4 @@ def ais_1_2_3_decode(body, pad): - try: - return libais.decode(body[:28], 0) - except DecodeError as e: - raise DecodeError(f'TYPE 1-2-3 LIBAIS ERR: {str(e)}') - + return libais.decode(body[:28], 0) diff --git a/tests/test_aivdm.py b/tests/test_aivdm.py index 83a2433..c746988 100644 --- a/tests/test_aivdm.py +++ b/tests/test_aivdm.py @@ -33,7 +33,8 @@ def test_decode(nmea, expected): @pytest.mark.parametrize("nmea,error", [ ('!AIVDM,2,1,1,B,@,0*57', 'Expected 2 message parts to decode but found 1'), ('!', 'No valid AIVDM found in'), - ('!AIVDM,1,1,,A,B99999,0*5D', 'AISTOOLS ERR: Not enough bits to decode. Need at least 149 bits, got only 36') + ('!AIVDM,1,1,,A,B99999,0*5D', 'AISTOOLS ERR: Not enough bits to decode. Need at least 149 bits, got only 36'), + ('!AIVDM,1,1,,A,1000,0*28', 'AISTOOLS ERR: Ais1_2_3: AIS_ERR_BAD_BIT_COUNT'), ]) def test_decode_fail(nmea, error): decoder = AIVDM() From ec18ecccbee048fb4f93f241b0a769b002a7d77e Mon Sep 17 00:00:00 2001 From: Paul Woods Date: Thu, 6 Jun 2024 20:37:46 -0400 Subject: [PATCH 11/16] bump dev version number --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4df1065..cfc5229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7.dev3" +version = "v0.1.7.dev4" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"}, From 40d23e7ccff3c4f8d3c8c775c186a721e2777d80 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Tue, 19 Nov 2024 20:59:10 -0500 Subject: [PATCH 12/16] fix #69 update for compatibility with bitarray 3.0.0 --- ais_tools/ais25.py | 2 +- ais_tools/transcode.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ais_tools/ais25.py b/ais_tools/ais25.py index 992e6ce..ed87e03 100644 --- a/ais_tools/ais25.py +++ b/ais_tools/ais25.py @@ -22,7 +22,7 @@ def ais25_decode(body, pad): # assume that the pad value was wrong and just ignore the extra bits at the end new_len = (len(text_bits) // 6) * 6 text_bits = text_bits[:new_len] - message['text'] = ''.join(text_bits.iterdecode(ASCII8toASCII6_decode_tree)) + message['text'] = ''.join(text_bits.decode(ASCII8toASCII6_decode_tree)) return message diff --git a/ais_tools/transcode.py b/ais_tools/transcode.py index 04926b5..43ed333 100644 --- a/ais_tools/transcode.py +++ b/ais_tools/transcode.py @@ -30,7 +30,7 @@ def bits_to_nmea(bits): pad = 0 else: bits = bits + (pad * bitarray('0')) - return ''.join(bits.iterdecode(ASCII8toAIS6_decode_tree)), pad + return ''.join(bits.decode(ASCII8toAIS6_decode_tree)), pad def nmea_to_bits(body, pad): @@ -73,7 +73,7 @@ def to_nmea(self): bits = self.bits else: bits = self.bits + (pad * bitarray('0')) - return ''.join(bits.iterdecode(ASCII8toAIS6_decode_tree)), pad + return ''.join(bits.decode(ASCII8toAIS6_decode_tree)), pad def pack(self, struct, message): self.pack_into(struct, self.offset, message) @@ -225,4 +225,4 @@ def decode(self, value): bits = bitarray() bits.frombytes(value) bits = bits[:self.nbits] - return ''.join(bits.iterdecode(ASCII8toASCII6_decode_tree)) + return ''.join(bits.decode(ASCII8toASCII6_decode_tree)) From 0a7b66b8adadebd92389be74d933f53034eb29f8 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Tue, 19 Nov 2024 21:33:11 -0500 Subject: [PATCH 13/16] change CI to build with macos-latest instead of macos-11 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 456e176..6f68c4a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: os: - - macos-11 + - macos-latest - ubuntu-latest python-version: - "3.8" From 39d92491bc7253aaf8b134f17d49fd80d48087fe Mon Sep 17 00:00:00 2001 From: Paul Woods Date: Tue, 19 Nov 2024 21:37:06 -0500 Subject: [PATCH 14/16] Update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cfc5229..5576145 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7.dev4" +version = "v0.1.7.dev5" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"}, From 2affaba5af04f4070b2051c3e14b57fc18398d91 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Wed, 20 Nov 2024 17:16:21 -0500 Subject: [PATCH 15/16] limit bitarray version to >=3.0.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cfc5229..e43b638 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "Click", "gpxpy", "requests", - "bitarray", + "bitarray>=3.0.0", "cbitstruct", "udatetime", "xxhash" From a0c46a756668ec089b29a10f644b332ad4647ac6 Mon Sep 17 00:00:00 2001 From: pwoods25443 Date: Wed, 20 Nov 2024 17:21:02 -0500 Subject: [PATCH 16/16] change to release version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 69ae731..e863fef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "ais-tools" description = "Tools for managing AIS messages" readme = "README.md" -version = "v0.1.7.dev5" +version = "v0.1.7" license = {file = "LICENSE"} authors = [ {name = "Paul Woods", email = "paul@globalfishingwatch.org"},