diff --git a/docsrc/conf.py b/docsrc/conf.py index 42949f631..3b1759eb2 100644 --- a/docsrc/conf.py +++ b/docsrc/conf.py @@ -96,6 +96,7 @@ 'pytest-cov', 'pytest-timeout', 'redis', + 'quantiphy', 'requests', 'rich', 'semver', diff --git a/src/analysis/plugin/compat.py b/src/analysis/plugin/compat.py index 1ebf3d495..e624f9615 100644 --- a/src/analysis/plugin/compat.py +++ b/src/analysis/plugin/compat.py @@ -58,19 +58,17 @@ def shutdown(self): def yara_match_to_dict(match: yara.Match) -> dict: """Converts a ``yara.Match`` to the format that :py:class:`analysis.YaraPluginBase` would return.""" - # FIXME (yara): Use this when we upgrade to yara-python 4.3.0 - # for string_match in match.strings: - # for string_instance in string_match.instances: - - strings = [(offset, identifier, data.hex()) for offset, identifier, data in match.strings] + # see YARA docs: https://yara.readthedocs.io/en/latest/yarapython.html#yara.StringMatchInstance + strings = [ + (string_instance.offset, string_match.identifier, string_instance.matched_data.decode(errors='replace')) + for string_match in match.strings # type: yara.StringMatch + for string_instance in string_match.instances # type: yara.StringMatchInstance + ] return { 'meta': { - # Optional - 'date': match.meta.get('date'), - # Optional - 'author': match.meta.get('author'), - 'description': match.meta['description'], + key: match.meta.get(key) + for key in ('open_source', 'software_name', 'website', 'date', 'author', 'description') }, 'rule': match.rule, 'strings': strings, diff --git a/src/helperFunctions/web_interface.py b/src/helperFunctions/web_interface.py index 98f91156d..42abec56e 100644 --- a/src/helperFunctions/web_interface.py +++ b/src/helperFunctions/web_interface.py @@ -7,7 +7,7 @@ from common_helper_files import get_binary_from_file from matplotlib import cm, colors from passlib.context import CryptContext -from si_prefix import si_format +from quantiphy import Quantity from helperFunctions.fileSystem import get_template_dir @@ -113,7 +113,9 @@ def cap_length_of_element(hid_element: str, maximum: int = 55) -> str: def _format_si_prefix(number: float, unit: str) -> str: - return f'{si_format(number, precision=2)}{unit}' + quantity = Quantity(number, unit) + quantity.set_prefs(map_sf={'u': 'µ'}) + return quantity.render(prec=2) def format_time(seconds: float) -> str: diff --git a/src/install/backend.py b/src/install/backend.py index 726a3792d..913cdb7db 100644 --- a/src/install/backend.py +++ b/src/install/backend.py @@ -120,7 +120,7 @@ def _install_plugins(distribution, skip_docker, only_docker=False): def _install_yara(): - yara_version = 'v4.2.3' # must be the same version as `yara-python` in `install/requirements_common.txt` + yara_version = 'v4.4.0' # must be the same version as `yara-python` in `install/requirements_common.txt` yara_process = subprocess.run('yara --version', shell=True, stdout=PIPE, stderr=STDOUT, text=True) if yara_process.returncode == 0 and yara_process.stdout.strip() == yara_version.strip('v'): diff --git a/src/install/requirements_backend.txt b/src/install/requirements_backend.txt index a17066d32..160d31a6c 100644 --- a/src/install/requirements_backend.txt +++ b/src/install/requirements_backend.txt @@ -1,5 +1,5 @@ cryptography==41.0.6 -docker==6.0.1 +docker~=6.1.3 MarkupSafe==2.1.1 networkx==2.6.3 Pillow==10.0.1 @@ -13,4 +13,4 @@ flask==2.3.3 git+https://github.com/fkie-cad/common_helper_yara.git # For plugin definition -semver==3.0.1 +semver==3.0.2 diff --git a/src/install/requirements_common.txt b/src/install/requirements_common.txt index 703894d82..c7e643212 100644 --- a/src/install/requirements_common.txt +++ b/src/install/requirements_common.txt @@ -1,9 +1,6 @@ # General build stuff testresources==2.0.1 -# FixMe: remove pinned version as soon as installation of ssdeep works again -setuptools<66 - # General python dependencies appdirs==1.4.4 flaky==3.7.0 @@ -22,7 +19,8 @@ rich==12.6.0 sqlalchemy==2.0.15 ssdeep==3.4 xmltodict==0.13.0 -yara-python==4.2.3 +# FixMe: pin to 4.4.x as soon as it releases +git+https://github.com/VirusTotal/yara-python@d0921c0 # Config validation pydantic==2.1.1 diff --git a/src/install/requirements_frontend.txt b/src/install/requirements_frontend.txt index e4908f7eb..536024ab0 100644 --- a/src/install/requirements_frontend.txt +++ b/src/install/requirements_frontend.txt @@ -8,12 +8,12 @@ flask==2.3.3 flask_restx==1.1.0 flask_sqlalchemy==3.0.5 itsdangerous==2.1.2 -matplotlib==3.5.3 +matplotlib==3.7.3 more_itertools==9.0.0 -prompt_toolkit==3.0.32 +prompt_toolkit==3.0.41 python-dateutil==2.8.2 -si-prefix==1.2.2 -uwsgi==2.0.22 +quantiphy==2.19 +uwsgi==2.0.23 virtualenv # must be below dependent packages (flask, flask-login, flask-restx) @@ -23,4 +23,4 @@ werkzeug==2.3.8 bleach==5.0.1 # Figuring out if the analysis is outdated -semver==3.0.1 +semver==3.0.2 diff --git a/src/plugins/analysis/binwalk/requirements.txt b/src/plugins/analysis/binwalk/requirements.txt index f375fc1cb..cb82452ad 100644 --- a/src/plugins/analysis/binwalk/requirements.txt +++ b/src/plugins/analysis/binwalk/requirements.txt @@ -1,3 +1,3 @@ capstone==4.0.2 cstruct==4.0 -matplotlib==3.5.3 +matplotlib==3.7.3 diff --git a/src/plugins/analysis/cve_lookup/requirements.txt b/src/plugins/analysis/cve_lookup/requirements.txt index 1b92350d1..549eda595 100644 --- a/src/plugins/analysis/cve_lookup/requirements.txt +++ b/src/plugins/analysis/cve_lookup/requirements.txt @@ -1,2 +1 @@ -pyxdameraulevenshtein==1.7.1 retry==0.9.2 diff --git a/src/plugins/analysis/ip_and_uri_finder/requirements.txt b/src/plugins/analysis/ip_and_uri_finder/requirements.txt index 7f7c7e8af..1e4eab0ba 100644 --- a/src/plugins/analysis/ip_and_uri_finder/requirements.txt +++ b/src/plugins/analysis/ip_and_uri_finder/requirements.txt @@ -1,2 +1,4 @@ git+https://github.com/fkie-cad/common_analysis_ip_and_uri.git -geoip2==4.6.0 +geoip2==4.7.0 +# dependency of geoip2 for python >= 3.12 +aiohttp~=3.9.0 diff --git a/src/test/unit/analysis/test_addons_yara.py b/src/test/unit/analysis/test_addons_yara.py new file mode 100644 index 000000000..8b77ad1b3 --- /dev/null +++ b/src/test/unit/analysis/test_addons_yara.py @@ -0,0 +1,52 @@ +from io import FileIO +from pathlib import Path + +import yara + +from analysis.YaraPluginBase import YaraBasePlugin +from analysis.plugin.addons import Yara +from analysis.plugin.compat import yara_match_to_dict +from helperFunctions.fileSystem import get_src_dir +from test.common_helper import create_test_file_object + +signature_file = str(Path(get_src_dir()) / 'test/unit/analysis/test.yara') +test_target = str(Path(get_src_dir()) / 'test/data/files/get_files_test/testfile1') + +EXPECTED_RESULT = { + 'matches': True, + 'meta': { + 'description': 'Generic Software', + 'open_source': False, + 'software_name': 'Test Software', + 'website': 'http://www.fkie.fraunhofer.de', + }, + 'rule': 'testRule', + 'strings': [(0, '$a', 'test'), (22, '$a', 'Test')], +} + + +class MockYaraPlugin(YaraBasePlugin): + def __init__(self): + self.signature_path = signature_file + self.NAME = 'test_plugin' + + +class MockYaraAddonPlugin(Yara): + def __init__(self): + self._rules = yara.compile(signature_file) + + +def test_output_is_compatible(): + fo = create_test_file_object(test_target) + plugin = MockYaraPlugin() + plugin.process_object(fo) + assert fo.processed_analysis['test_plugin']['testRule'] == EXPECTED_RESULT + + yara_addon_plugin = MockYaraAddonPlugin() + file = FileIO(test_target) + yara_matches = yara_addon_plugin.match(file) + assert all(isinstance(m, yara.Match) for m in yara_matches) + converted_match = yara_match_to_dict(yara_matches[0]) + assert converted_match['strings'] == EXPECTED_RESULT['strings'] + for key, value in EXPECTED_RESULT['meta'].items(): + assert converted_match['meta'][key] == value diff --git a/src/test/unit/helperFunctions/test_web_interface.py b/src/test/unit/helperFunctions/test_web_interface.py index 44a5ce8be..ab414d82a 100644 --- a/src/test/unit/helperFunctions/test_web_interface.py +++ b/src/test/unit/helperFunctions/test_web_interface.py @@ -74,9 +74,9 @@ def test_cap_length_of_element_short(): @pytest.mark.parametrize( ('number', 'unit', 'expected_output'), [ - (1, 'm', '1.00 m'), - (0.034, 'g', '34.00 mg'), - (0.0000123456789, 's', '12.35 µs'), + (1, 'm', '1 m'), + (0.034, 'g', '34 mg'), + (0.0000123456789, 's', '12.3 µs'), (1234.5, 'm', '1.23 km'), ], ) @@ -87,8 +87,8 @@ def test_format_si_prefix(number, unit, expected_output): @pytest.mark.parametrize( ('seconds', 'expected_output'), [ - (2, '2.00 s'), - (0.2, '200.00 ms'), + (2, '2 s'), + (0.2, '200 ms'), (120, '0:02:00'), (100000, '1 day, 3:46:40'), ],