Created base aql platform and fixes

Author: nazargesyk

uncoder-core/app/translator/core/exceptions/core.py

@@ -77,3 +77,7 @@ class InvalidYamlStructure(InvalidRuleStructure):
 
 class InvalidJSONStructure(InvalidRuleStructure):
     rule_type: str = "JSON"
+
+
+class InvalidXMLStructure(InvalidRuleStructure):
+    rule_type: str = "XML"

uncoder-core/app/translator/core/mixins/rule.py

@@ -1,8 +1,10 @@
 import json
+from typing import Union
 
+import xmltodict
 import yaml
 
-from app.translator.core.exceptions.core import InvalidJSONStructure, InvalidYamlStructure
+from app.translator.core.exceptions.core import InvalidJSONStructure, InvalidXMLStructure, InvalidYamlStructure
 from app.translator.core.mitre import MitreConfig
 
 
@@ -36,5 +38,13 @@ def parse_mitre_attack(self, tags: list[str]) -> dict[str, list]:
                     result["techniques"].append(technique)
             elif tactic := self.mitre_config.get_tactic(tag):
                 result["tactics"].append(tactic)
-
         return result
+
+
+class XMLRuleMixin:
+    @staticmethod
+    def load_rule(text: Union[str, bytes]) -> dict:
+        try:
+            return xmltodict.parse(text)
+        except Exception as err:
+            raise InvalidXMLStructure(error=str(err)) from err

uncoder-core/app/translator/core/models/query_container.py

@@ -56,6 +56,13 @@ class RawQueryContainer:
     meta_info: MetaInfoContainer = field(default_factory=MetaInfoContainer)
 
 
+@dataclass
+class RawQueryDictContainer:
+    query: dict
+    language: str
+    meta_info: dict
+
+
 @dataclass
 class TokenizedQueryContainer:
     tokens: list[TOKEN_TYPE]

uncoder-core/app/translator/core/parser.py

@@ -32,9 +32,13 @@
 
 class QueryParser(ABC):
     wrapped_with_comment_pattern: str = None
+    details: PlatformDetails = None
 
     def remove_comments(self, text: str) -> str:
-        return re.sub(self.wrapped_with_comment_pattern, "\n", text, flags=re.MULTILINE).strip()
+        if self.wrapped_with_comment_pattern:
+            return re.sub(self.wrapped_with_comment_pattern, "\n", text, flags=re.MULTILINE).strip()
+
+        return text
 
     def parse_raw_query(self, text: str, language: str) -> RawQueryContainer:
         return RawQueryContainer(query=text, language=language)
@@ -47,7 +51,6 @@ def parse(self, raw_query_container: RawQueryContainer) -> TokenizedQueryContain
 class PlatformQueryParser(QueryParser, ABC):
     mappings: BasePlatformMappings = None
     tokenizer: QueryTokenizer = None
-    details: PlatformDetails = None
     platform_functions: PlatformFunctions = None
 
     def get_fields_tokens(self, tokens: list[Union[FieldValue, Keyword, Identifier]]) -> list[Field]:

uncoder-core/app/translator/core/render_cti.py

@@ -19,6 +19,7 @@
 
 
 from app.translator.core.models.iocs import IocsChunkValue
+from app.translator.core.models.platform_details import PlatformDetails
 
 
 class RenderCTI:
@@ -31,6 +32,7 @@ class RenderCTI:
     final_result_for_many: str = "union * | where ({result})\n"
     final_result_for_one: str = "union * | where {result}\n"
     default_mapping = None
+    details: PlatformDetails = None
 
     def create_field_value(self, field: str, value: str, generic_field: str) -> str:  # noqa: ARG002
         return self.field_value_template.format(key=field, value=value)

uncoder-core/app/translator/core/tokenizer.py

@@ -52,6 +52,8 @@ class QueryTokenizer(BaseTokenizer):
     single_value_operators_map: ClassVar[dict[str, str]] = {}
     # used to generate re pattern. so the keys order is important
     multi_value_operators_map: ClassVar[dict[str, str]] = {}
+    # used to generate re pattern. so the keys order is important
+    fields_operator_map: ClassVar[dict[str, str]] = {}
     operators_map: ClassVar[dict[str, str]] = {}  # used to generate re pattern. so the keys order is important
 
     logical_operator_pattern = r"^(?P<logical_operator>and|or|not|AND|OR|NOT)\s+"
@@ -73,7 +75,11 @@ class QueryTokenizer(BaseTokenizer):
     def __init_subclass__(cls, **kwargs):
         cls._validate_re_patterns()
         cls.value_pattern = cls.base_value_pattern.replace("___value_pattern___", cls._value_pattern)
-        cls.operators_map = {**cls.single_value_operators_map, **cls.multi_value_operators_map}
+        cls.operators_map = {
+            **cls.single_value_operators_map,
+            **cls.multi_value_operators_map,
+            **cls.fields_operator_map,
+        }
         cls.operator_pattern = rf"""(?:___field___\s*(?P<operator>(?:{'|'.join(cls.operators_map)})))\s*"""
 
     @classmethod

uncoder-core/app/translator/platforms/arcsight/__init__.py

@@ -1 +1 @@
-from app.translator.platforms.arcsight.renders.arcsight_cti import ArcsightKeyword
+from app.translator.platforms.arcsight.renders.arcsight_cti import ArcsightKeyword  # noqa: F401

uncoder-core/app/translator/platforms/arcsight/const.py

@@ -0,0 +1,8 @@
+ARCSIGHT_QUERY_DETAILS = {
+    "platform_id": "arcsight",
+    "name": "ArcSight Query",
+    "group_name": "ArcSight",
+    "group_id": "arcsight",
+    "platform_name": "Query",
+    "alt_platform_name": "CEF",
+}

uncoder-core/app/translator/platforms/arcsight/mappings/__init__.py

@@ -0,0 +1,12 @@
+DEFAULT_ARCSIGHT_MAPPING = {
+    "SourceIP": "sourceAddress",
+    "DestinationIP": "destinationAddress",
+    "Domain": "destinationDnsDomain",
+    "URL": "requestUrl",
+    "HashMd5": "fileHash",
+    "HashSha1": "fileHash",
+    "HashSha256": "fileHash",
+    "HashSha512": "fileHash",
+    "Emails": "sender-address",
+    "Files": "winlog.event_data.TargetFilename",
+}

uncoder-core/app/translator/platforms/arcsight/mappings/arcsight_cti.py

@@ -1,3 +1,3 @@
-from app.translator.platforms.athena.parsers.athena import AthenaQueryParser
-from app.translator.platforms.athena.renders.athena import AthenaQueryRender
-from app.translator.platforms.athena.renders.athena_cti import AthenaCTI
+from app.translator.platforms.athena.parsers.athena import AthenaQueryParser  # noqa: F401
+from app.translator.platforms.athena.renders.athena import AthenaQueryRender  # noqa: F401
+from app.translator.platforms.athena.renders.athena_cti import AthenaCTI  # noqa: F401

uncoder-core/app/translator/platforms/arcsight/renders/__init__.py

@@ -0,0 +1,3 @@
+UTF8_PAYLOAD_PATTERN = r"UTF8\(payload\)"
+NUM_VALUE_PATTERN = r"(?P<num_value>\d+(?:\.\d+)*)"
+SINGLE_QUOTES_VALUE_PATTERN = r"""'(?P<s_q_value>(?:[:a-zA-Z\*0-9=+%#\-\/\\,_".$&^@!\(\)\{\}\s]|'')*)'"""

uncoder-core/app/translator/platforms/athena/__init__.py

@@ -0,0 +1,8 @@
+from app.translator.core.escape_manager import EscapeManager
+
+
+class AQLEscapeManager(EscapeManager):
+    ...
+
+
+aql_escape_manager = AQLEscapeManager()

uncoder-core/app/translator/platforms/base/aql/__init__.py

@@ -0,0 +1,122 @@
+"""
+Uncoder IO Community Edition License
+-----------------------------------------------------------------
+Copyright (c) 2024 SOC Prime, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-----------------------------------------------------------------
+"""
+from typing import Union
+
+from app.translator.const import DEFAULT_VALUE_TYPE
+from app.translator.core.custom_types.values import ValueType
+from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender
+from app.translator.platforms.base.aql.escape_manager import aql_escape_manager
+from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings
+
+
+class AQLFieldValue(BaseQueryFieldValue):
+    escape_manager = aql_escape_manager
+
+    def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) -> Union[str, int]:  # noqa: ARG002
+        if isinstance(value, str):
+            value = value.replace("_", "__").replace("%", "%%").replace("\\'", "%").replace("'", '"')
+            if value.endswith("\\\\%"):
+                value = value.replace("\\\\%", "\\%")
+        return value
+
+    def _apply_value(self, value: Union[str, int]) -> Union[str, int]:
+        if isinstance(value, str) and "\\" in value:
+            return value
+        return self.apply_value(value)
+
+    def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})"
+        if field == "UTF8(payload)":
+            return f"UTF8(payload) ILIKE '{self.apply_value(value)}'"
+        if isinstance(value, int):
+            return f'"{field}"={value}'
+
+        return f"\"{field}\"='{self._apply_value(value)}'"
+
+    def less_modifier(self, field: str, value: Union[int, str]) -> str:
+        if isinstance(value, int):
+            return f'"{field}"<{value}'
+        return f"\"{field}\"<'{self._apply_value(value)}'"
+
+    def less_or_equal_modifier(self, field: str, value: Union[int, str]) -> str:
+        if isinstance(value, int):
+            return f'"{field}"<={value}'
+        return f"\"{field}\"<='{self._apply_value(value)}'"
+
+    def greater_modifier(self, field: str, value: Union[int, str]) -> str:
+        if isinstance(value, int):
+            return f'"{field}">{value}'
+        return f"\"{field}\">'{self._apply_value(value)}'"
+
+    def greater_or_equal_modifier(self, field: str, value: Union[int, str]) -> str:
+        if isinstance(value, int):
+            return f'"{field}">={value}'
+        return f"\"{field}\">='{self._apply_value(value)}'"
+
+    def not_equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join([self.not_equal_modifier(field=field, value=v) for v in value])})"
+        if isinstance(value, int):
+            return f'"{field}"!={value}'
+        return f"\"{field}\"!='{self._apply_value(value)}'"
+
+    def contains_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})"
+        return f"\"{field}\" ILIKE '%{self._apply_value(value)}%'"
+
+    def endswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.endswith_modifier(field=field, value=v) for v in value)})"
+        return f"\"{field}\" ILIKE '%{self._apply_value(value)}'"
+
+    def startswith_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.startswith_modifier(field=field, value=v) for v in value)})"
+        return f"\"{field}\" ILIKE '{self._apply_value(value)}%'"
+
+    def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.regex_modifier(field=field, value=v) for v in value)})"
+        return f"\"{field}\" IMATCHES '{value}'"
+
+    def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.keywords(field=field, value=v) for v in value)})"
+        return f"UTF8(payload) ILIKE '%{self.apply_value(value)}%'"
+
+
+class AQLQueryRender(PlatformQueryRender):
+    mappings: AQLMappings = aql_mappings
+
+    or_token = "OR"
+    and_token = "AND"
+    not_token = "NOT"
+
+    field_value_map = AQLFieldValue(or_token=or_token)
+    query_pattern = "{prefix} AND {query} {functions}"
+
+    def generate_prefix(self, log_source_signature: AQLLogSourceSignature) -> str:
+        table = str(log_source_signature)
+        extra_condition = log_source_signature.extra_condition
+        return f"SELECT UTF8(payload) FROM {table} WHERE {extra_condition}"
+
+    def wrap_with_comment(self, value: str) -> str:
+        return f"/* {value} */"

uncoder-core/app/translator/platforms/base/aql/const.py

@@ -15,7 +15,6 @@
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 -----------------------------------------------------------------
 """
-
 import re
 from typing import Any, ClassVar, Union
 
@@ -24,12 +23,12 @@
 from app.translator.core.models.field import FieldValue, Keyword
 from app.translator.core.models.identifier import Identifier
 from app.translator.core.tokenizer import QueryTokenizer
-from app.translator.platforms.qradar.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, UTF8_PAYLOAD_PATTERN
-from app.translator.platforms.qradar.escape_manager import qradar_escape_manager
+from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN, UTF8_PAYLOAD_PATTERN
+from app.translator.platforms.base.aql.escape_manager import aql_escape_manager
 from app.translator.tools.utils import get_match_group
 
 
-class QradarTokenizer(QueryTokenizer):
+class AQLTokenizer(QueryTokenizer):
     single_value_operators_map: ClassVar[dict[str, str]] = {
         "=": OperatorType.EQ,
         "<=": OperatorType.LTE,
@@ -49,7 +48,7 @@ class QradarTokenizer(QueryTokenizer):
     _value_pattern = rf"{NUM_VALUE_PATTERN}|{bool_value_pattern}|{SINGLE_QUOTES_VALUE_PATTERN}"
     multi_value_pattern = rf"""\((?P<{ValueType.multi_value}>[:a-zA-Z\"\*0-9=+%#\-_\/\\'\,.&^@!\(\s]*)\)"""
     keyword_pattern = rf"{UTF8_PAYLOAD_PATTERN}\s+(?:like|LIKE|ilike|ILIKE)\s+{SINGLE_QUOTES_VALUE_PATTERN}"
-    escape_manager = qradar_escape_manager
+    escape_manager = aql_escape_manager
 
     wildcard_symbol = "%"
 

uncoder-core/app/translator/platforms/base/aql/escape_manager.py

@@ -1 +1 @@
-from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI
+from app.translator.platforms.carbonblack.renders.carbonblack_cti import CarbonBlackCTI  # noqa: F401

uncoder-core/app/translator/platforms/base/aql/parsers/__init__.py

@@ -1,5 +1,5 @@
-from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser
-from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser
-from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender
-from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI
-from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender
+from app.translator.platforms.chronicle.parsers.chronicle import ChronicleQueryParser  # noqa: F401
+from app.translator.platforms.chronicle.parsers.chronicle_rule import ChronicleRuleParser  # noqa: F401
+from app.translator.platforms.chronicle.renders.chronicle import ChronicleQueryRender  # noqa: F401
+from app.translator.platforms.chronicle.renders.chronicle_cti import ChronicleQueryCTI  # noqa: F401
+from app.translator.platforms.chronicle.renders.chronicle_rule import ChronicleSecurityRuleRender  # noqa: F401

uncoder-core/app/translator/platforms/base/aql/renders/__init__.py

@@ -1,3 +1,3 @@
-from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeQueryParser
-from app.translator.platforms.crowdstrike.renders.crowdstrike import CrowdStrikeQueryRender
-from app.translator.platforms.crowdstrike.renders.crowdstrike_cti import CrowdStrikeCTI
+from app.translator.platforms.crowdstrike.parsers.crowdstrike import CrowdStrikeQueryParser  # noqa: F401
+from app.translator.platforms.crowdstrike.renders.crowdstrike import CrowdStrikeQueryRender  # noqa: F401
+from app.translator.platforms.crowdstrike.renders.crowdstrike_cti import CrowdStrikeCTI  # noqa: F401

uncoder-core/app/translator/platforms/base/aql/renders/aql.py

@@ -1,8 +1,8 @@
-from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser
-from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser
-from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender
-from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender
-from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender
-from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI
-from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender
-from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender
+from app.translator.platforms.elasticsearch.parsers.detection_rule import ElasticSearchRuleParser  # noqa: F401
+from app.translator.platforms.elasticsearch.parsers.elasticsearch import ElasticSearchQueryParser  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.detection_rule import ElasticSearchRuleRender  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.elast_alert import ElastAlertRuleRender  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.elasticsearch import ElasticSearchQueryRender  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.elasticsearch_cti import ElasticsearchCTI  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.kibana import KibanaRuleRender  # noqa: F401
+from app.translator.platforms.elasticsearch.renders.xpack_watcher import XPackWatcherRuleRender  # noqa: F401

uncoder-core/app/translator/platforms/qradar/tokenizer.py renamed to uncoder-core/app/translator/platforms/base/aql/tokenizer.py

@@ -1 +1 @@
-from app.translator.platforms.fireeye_helix.renders.fireeye_helix_cti import FireeyeHelixCTI
+from app.translator.platforms.fireeye_helix.renders.fireeye_helix_cti import FireeyeHelixCTI  # noqa: F401

uncoder-core/app/translator/platforms/carbonblack/__init__.py

@@ -1 +1 @@
-from app.translator.platforms.forti_siem.renders.forti_siem_rule import FortiSiemRuleRender
+from app.translator.platforms.forti_siem.renders.forti_siem_rule import FortiSiemRuleRender  # noqa: F401

uncoder-core/app/translator/platforms/chronicle/__init__.py

@@ -1,3 +1,3 @@
-from app.translator.platforms.graylog.parsers.graylog import GraylogQueryParser
-from app.translator.platforms.graylog.renders.graylog import GraylogQueryRender
-from app.translator.platforms.graylog.renders.graylog_cti import GraylogCTI
+from app.translator.platforms.graylog.parsers.graylog import GraylogQueryParser  # noqa: F401
+from app.translator.platforms.graylog.renders.graylog import GraylogQueryRender  # noqa: F401
+from app.translator.platforms.graylog.renders.graylog_cti import GraylogCTI  # noqa: F401

uncoder-core/app/translator/platforms/crowdstrike/__init__.py

@@ -1 +1 @@
-from app.translator.platforms.logpoint.renders.logpoint_cti import LogpointCTI
+from app.translator.platforms.logpoint.renders.logpoint_cti import LogpointCTI  # noqa: F401

uncoder-core/app/translator/platforms/elasticsearch/__init__.py

@@ -1,2 +1,2 @@
-from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import LogRhythmAxonQueryRender
-from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_rule import LogRhythmAxonRuleRender
+from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import LogRhythmAxonQueryRender  # noqa: F401
+from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_rule import LogRhythmAxonRuleRender  # noqa: F401

uncoder-core/app/translator/platforms/fireeye_helix/__init__.py

@@ -1,5 +1,5 @@
-from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser
-from app.translator.platforms.logscale.parsers.logscale_alert import LogScaleAlertParser
-from app.translator.platforms.logscale.renders.logscale import LogScaleQueryRender
-from app.translator.platforms.logscale.renders.logscale_alert import LogScaleAlertRender
-from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI
+from app.translator.platforms.logscale.parsers.logscale import LogScaleQueryParser  # noqa: F401
+from app.translator.platforms.logscale.parsers.logscale_alert import LogScaleAlertParser  # noqa: F401
+from app.translator.platforms.logscale.renders.logscale import LogScaleQueryRender  # noqa: F401
+from app.translator.platforms.logscale.renders.logscale_alert import LogScaleAlertRender  # noqa: F401
+from app.translator.platforms.logscale.renders.logscale_cti import LogScaleCTI  # noqa: F401