From a547ba17351efd29d8673ada255931c044cda493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 10 Oct 2023 12:10:18 +0200 Subject: [PATCH 1/9] replaced abandoned package 'si-prefix' with 'quantiphy' --- src/helperFunctions/web_interface.py | 6 ++++-- src/install/requirements_frontend.txt | 2 +- src/test/unit/helperFunctions/test_web_interface.py | 10 +++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/helperFunctions/web_interface.py b/src/helperFunctions/web_interface.py index 98f91156d..42abec56e 100644 --- a/src/helperFunctions/web_interface.py +++ b/src/helperFunctions/web_interface.py @@ -7,7 +7,7 @@ from common_helper_files import get_binary_from_file from matplotlib import cm, colors from passlib.context import CryptContext -from si_prefix import si_format +from quantiphy import Quantity from helperFunctions.fileSystem import get_template_dir @@ -113,7 +113,9 @@ def cap_length_of_element(hid_element: str, maximum: int = 55) -> str: def _format_si_prefix(number: float, unit: str) -> str: - return f'{si_format(number, precision=2)}{unit}' + quantity = Quantity(number, unit) + quantity.set_prefs(map_sf={'u': 'µ'}) + return quantity.render(prec=2) def format_time(seconds: float) -> str: diff --git a/src/install/requirements_frontend.txt b/src/install/requirements_frontend.txt index 9c61e2e88..ed4d3b9e2 100644 --- a/src/install/requirements_frontend.txt +++ b/src/install/requirements_frontend.txt @@ -12,7 +12,7 @@ matplotlib==3.5.3 more_itertools==9.0.0 prompt_toolkit==3.0.32 python-dateutil==2.8.2 -si-prefix==1.2.2 +quantiphy==2.19 uwsgi==2.0.22 virtualenv diff --git a/src/test/unit/helperFunctions/test_web_interface.py b/src/test/unit/helperFunctions/test_web_interface.py index 44a5ce8be..ab414d82a 100644 --- a/src/test/unit/helperFunctions/test_web_interface.py +++ b/src/test/unit/helperFunctions/test_web_interface.py @@ -74,9 +74,9 @@ def test_cap_length_of_element_short(): @pytest.mark.parametrize( ('number', 'unit', 'expected_output'), [ - (1, 'm', '1.00 m'), - (0.034, 'g', '34.00 mg'), - (0.0000123456789, 's', '12.35 µs'), + (1, 'm', '1 m'), + (0.034, 'g', '34 mg'), + (0.0000123456789, 's', '12.3 µs'), (1234.5, 'm', '1.23 km'), ], ) @@ -87,8 +87,8 @@ def test_format_si_prefix(number, unit, expected_output): @pytest.mark.parametrize( ('seconds', 'expected_output'), [ - (2, '2.00 s'), - (0.2, '200.00 ms'), + (2, '2 s'), + (0.2, '200 ms'), (120, '0:02:00'), (100000, '1 day, 3:46:40'), ], From 5879f2752b82cc6b4630ac883d373e31252dadb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 10 Oct 2023 15:47:04 +0200 Subject: [PATCH 2/9] update matplotlib version and remove setuptools pin ssdeep installation should again work with the latest version of setuptools version 3.8 of matplotlib cannot be used yet, because it doesn't support Python3.8 --- src/install/requirements_common.txt | 3 --- src/install/requirements_frontend.txt | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/install/requirements_common.txt b/src/install/requirements_common.txt index 703894d82..f1c865110 100644 --- a/src/install/requirements_common.txt +++ b/src/install/requirements_common.txt @@ -1,9 +1,6 @@ # General build stuff testresources==2.0.1 -# FixMe: remove pinned version as soon as installation of ssdeep works again -setuptools<66 - # General python dependencies appdirs==1.4.4 flaky==3.7.0 diff --git a/src/install/requirements_frontend.txt b/src/install/requirements_frontend.txt index ed4d3b9e2..7c730ae6c 100644 --- a/src/install/requirements_frontend.txt +++ b/src/install/requirements_frontend.txt @@ -8,7 +8,7 @@ flask==2.3.3 flask_restx==1.1.0 flask_sqlalchemy==3.0.5 itsdangerous==2.1.2 -matplotlib==3.5.3 +matplotlib==3.7.3 more_itertools==9.0.0 prompt_toolkit==3.0.32 python-dateutil==2.8.2 From f5d33b68d7584efe26923606f1879487ff4cb04a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 10 Oct 2023 16:31:06 +0200 Subject: [PATCH 3/9] added new package 'quantiphy' to sphinx mock list --- docsrc/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docsrc/conf.py b/docsrc/conf.py index 42949f631..3b1759eb2 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -96,6 +96,7 @@ 'pytest-cov', 'pytest-timeout', 'redis', + 'quantiphy', 'requests', 'rich', 'semver', From f6473cf0e964521cb3a62cd35a8fdd7f03e4626b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Tue, 10 Oct 2023 16:38:46 +0200 Subject: [PATCH 4/9] removed now obsolete dependency 'pyxdameraulevenshtein' of cve_lookup --- src/plugins/analysis/cve_lookup/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/analysis/cve_lookup/requirements.txt b/src/plugins/analysis/cve_lookup/requirements.txt index 1b92350d1..549eda595 100644 --- a/src/plugins/analysis/cve_lookup/requirements.txt +++ b/src/plugins/analysis/cve_lookup/requirements.txt @@ -1,2 +1 @@ -pyxdameraulevenshtein==1.7.1 retry==0.9.2 From 976b0fbaf7d95682cb960a92c779701ccd3e097b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 22 Nov 2023 14:49:01 +0100 Subject: [PATCH 5/9] update remaining dependencies for python 3.12 and some additional dependencies which got new patches --- src/install/requirements_backend.txt | 2 +- src/install/requirements_frontend.txt | 6 +++--- src/plugins/analysis/ip_and_uri_finder/requirements.txt | 4 +++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/install/requirements_backend.txt b/src/install/requirements_backend.txt index 02b96a7d5..98e7d4fe0 100644 --- a/src/install/requirements_backend.txt +++ b/src/install/requirements_backend.txt @@ -13,4 +13,4 @@ flask==2.3.3 git+https://github.com/fkie-cad/common_helper_yara.git # For plugin definition -semver==3.0.1 +semver==3.0.2 diff --git a/src/install/requirements_frontend.txt b/src/install/requirements_frontend.txt index 7c730ae6c..b056b24f0 100644 --- a/src/install/requirements_frontend.txt +++ b/src/install/requirements_frontend.txt @@ -10,10 +10,10 @@ flask_sqlalchemy==3.0.5 itsdangerous==2.1.2 matplotlib==3.7.3 more_itertools==9.0.0 -prompt_toolkit==3.0.32 +prompt_toolkit==3.0.41 python-dateutil==2.8.2 quantiphy==2.19 -uwsgi==2.0.22 +uwsgi==2.0.23 virtualenv # must be below dependent packages (flask, flask-login, flask-restx) @@ -23,4 +23,4 @@ werkzeug==2.3.7 bleach==5.0.1 # Figuring out if the analysis is outdated -semver==3.0.1 +semver==3.0.2 diff --git a/src/plugins/analysis/ip_and_uri_finder/requirements.txt b/src/plugins/analysis/ip_and_uri_finder/requirements.txt index 7f7c7e8af..1e4eab0ba 100644 --- a/src/plugins/analysis/ip_and_uri_finder/requirements.txt +++ b/src/plugins/analysis/ip_and_uri_finder/requirements.txt @@ -1,2 +1,4 @@ git+https://github.com/fkie-cad/common_analysis_ip_and_uri.git -geoip2==4.6.0 +geoip2==4.7.0 +# dependency of geoip2 for python >= 3.12 +aiohttp~=3.9.0 From 67ac7c942ab1526988815d4f380e426b25fdc51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Fri, 24 Nov 2023 15:55:36 +0100 Subject: [PATCH 6/9] updated docker-py version to fix compatibility with newer requests --- src/install/requirements_backend.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/install/requirements_backend.txt b/src/install/requirements_backend.txt index 7750a1254..9be6a999b 100644 --- a/src/install/requirements_backend.txt +++ b/src/install/requirements_backend.txt @@ -1,5 +1,5 @@ cryptography==41.0.4 -docker==6.0.1 +docker~=6.1.3 MarkupSafe==2.1.1 networkx==2.6.3 Pillow==10.0.1 From 780d1e5d5ea6e88a17af6dc264896ffd84521544 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 13 Dec 2023 16:38:19 +0100 Subject: [PATCH 7/9] fixed matplotlib version mismatch --- src/plugins/analysis/binwalk/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/analysis/binwalk/requirements.txt b/src/plugins/analysis/binwalk/requirements.txt index f375fc1cb..cb82452ad 100644 --- a/src/plugins/analysis/binwalk/requirements.txt +++ b/src/plugins/analysis/binwalk/requirements.txt @@ -1,3 +1,3 @@ capstone==4.0.2 cstruct==4.0 -matplotlib==3.5.3 +matplotlib==3.7.3 From dcbf8027a5323394a2ca4de368d89aedfe09b3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 13 Dec 2023 16:38:53 +0100 Subject: [PATCH 8/9] upgrade yara version --- src/install/backend.py | 2 +- src/install/requirements_common.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/install/backend.py b/src/install/backend.py index 726a3792d..913cdb7db 100644 --- a/src/install/backend.py +++ b/src/install/backend.py @@ -120,7 +120,7 @@ def _install_plugins(distribution, skip_docker, only_docker=False): def _install_yara(): - yara_version = 'v4.2.3' # must be the same version as `yara-python` in `install/requirements_common.txt` + yara_version = 'v4.4.0' # must be the same version as `yara-python` in `install/requirements_common.txt` yara_process = subprocess.run('yara --version', shell=True, stdout=PIPE, stderr=STDOUT, text=True) if yara_process.returncode == 0 and yara_process.stdout.strip() == yara_version.strip('v'): diff --git a/src/install/requirements_common.txt b/src/install/requirements_common.txt index f1c865110..c7e643212 100644 --- a/src/install/requirements_common.txt +++ b/src/install/requirements_common.txt @@ -19,7 +19,8 @@ rich==12.6.0 sqlalchemy==2.0.15 ssdeep==3.4 xmltodict==0.13.0 -yara-python==4.2.3 +# FixMe: pin to 4.4.x as soon as it releases +git+https://github.com/VirusTotal/yara-python@d0921c0 # Config validation pydantic==2.1.1 From 96ce0ccec275650d4d0b33bfc209fc0143f08672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Wed, 20 Dec 2023 16:14:38 +0100 Subject: [PATCH 9/9] plugin compat: updated for new yara version + added test --- src/analysis/plugin/compat.py | 18 ++++---- src/test/unit/analysis/test_addons_yara.py | 52 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 src/test/unit/analysis/test_addons_yara.py diff --git a/src/analysis/plugin/compat.py b/src/analysis/plugin/compat.py index 1ebf3d495..e624f9615 100644 --- a/src/analysis/plugin/compat.py +++ b/src/analysis/plugin/compat.py @@ -58,19 +58,17 @@ def shutdown(self): def yara_match_to_dict(match: yara.Match) -> dict: """Converts a ``yara.Match`` to the format that :py:class:`analysis.YaraPluginBase` would return.""" - # FIXME (yara): Use this when we upgrade to yara-python 4.3.0 - # for string_match in match.strings: - # for string_instance in string_match.instances: - - strings = [(offset, identifier, data.hex()) for offset, identifier, data in match.strings] + # see YARA docs: https://yara.readthedocs.io/en/latest/yarapython.html#yara.StringMatchInstance + strings = [ + (string_instance.offset, string_match.identifier, string_instance.matched_data.decode(errors='replace')) + for string_match in match.strings # type: yara.StringMatch + for string_instance in string_match.instances # type: yara.StringMatchInstance + ] return { 'meta': { - # Optional - 'date': match.meta.get('date'), - # Optional - 'author': match.meta.get('author'), - 'description': match.meta['description'], + key: match.meta.get(key) + for key in ('open_source', 'software_name', 'website', 'date', 'author', 'description') }, 'rule': match.rule, 'strings': strings, diff --git a/src/test/unit/analysis/test_addons_yara.py b/src/test/unit/analysis/test_addons_yara.py new file mode 100644 index 000000000..8b77ad1b3 --- /dev/null +++ b/src/test/unit/analysis/test_addons_yara.py @@ -0,0 +1,52 @@ +from io import FileIO +from pathlib import Path + +import yara + +from analysis.YaraPluginBase import YaraBasePlugin +from analysis.plugin.addons import Yara +from analysis.plugin.compat import yara_match_to_dict +from helperFunctions.fileSystem import get_src_dir +from test.common_helper import create_test_file_object + +signature_file = str(Path(get_src_dir()) / 'test/unit/analysis/test.yara') +test_target = str(Path(get_src_dir()) / 'test/data/files/get_files_test/testfile1') + +EXPECTED_RESULT = { + 'matches': True, + 'meta': { + 'description': 'Generic Software', + 'open_source': False, + 'software_name': 'Test Software', + 'website': 'http://www.fkie.fraunhofer.de', + }, + 'rule': 'testRule', + 'strings': [(0, '$a', 'test'), (22, '$a', 'Test')], +} + + +class MockYaraPlugin(YaraBasePlugin): + def __init__(self): + self.signature_path = signature_file + self.NAME = 'test_plugin' + + +class MockYaraAddonPlugin(Yara): + def __init__(self): + self._rules = yara.compile(signature_file) + + +def test_output_is_compatible(): + fo = create_test_file_object(test_target) + plugin = MockYaraPlugin() + plugin.process_object(fo) + assert fo.processed_analysis['test_plugin']['testRule'] == EXPECTED_RESULT + + yara_addon_plugin = MockYaraAddonPlugin() + file = FileIO(test_target) + yara_matches = yara_addon_plugin.match(file) + assert all(isinstance(m, yara.Match) for m in yara_matches) + converted_match = yara_match_to_dict(yara_matches[0]) + assert converted_match['strings'] == EXPECTED_RESULT['strings'] + for key, value in EXPECTED_RESULT['meta'].items(): + assert converted_match['meta'][key] == value