Merge pull request #118 from UncoderIO/gis-7850

Improve AQL mapping logic; Palo Alto add support keywords; Sigma add …

Author: saltar-ua

uncoder-core/app/translator/mappings/platforms/sigma/windows_powershell.yml

@@ -5,6 +5,7 @@ description: Text that describe current mapping
 log_source:
   product: [windows]
   service: [powershell]
+  category: [ps_classic_provider_start, ps_classic_script, ps_classic_start, ps_module, ps_script]
 
 default_log_source:
   product: windows

uncoder-core/app/translator/mappings/platforms/sigma/windows_registry_event.yml

@@ -0,0 +1,31 @@
+platform: Sigma
+source: windows_registry_event
+
+log_source:
+  product: [windows]
+  category: [registry_event, registry_set]
+
+default_log_source:
+  product: windows
+  category: registry_event
+
+field_mapping:
+  TargetObject: TargetObject
+  Image: Image
+  Details: Details
+  EventType: EventType
+  CommandLine: CommandLine
+  LogonId: LogonId
+  Product: Product
+  Company: Company
+  IntegrityLevel: IntegrityLevel
+  CurrentDirectory: CurrentDirectory
+  ProcessId: ProcessId
+  ParentProcessId: ParentProcessId
+  ParentCommandLine: ParentCommandLine
+  ParentImage: ParentImage
+  ParentUser: ParentUser
+  ParentIntegrityLevel: ParentIntegrityLevel
+  ParentLogonId: ParentLogonId
+  ParentProduct: ParentProduct
+  ParentCompany: ParentCompany

uncoder-core/app/translator/platforms/base/aql/log_source_map.py

@@ -0,0 +1,68 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class AQLLogSourceMap:
+    name: str
+    id_map: dict[str, int]
+
+
+CATEGORYNAME_ID_MAP = {
+    "ACL Permit": 4012,
+    "Successful Registry Modification": 8012,
+    "File Created": 8028,
+    "Process Creation Success": 8110,
+    "DNS In Progress": 18081,
+    "Object Load Success": 19247,
+}
+
+DEVICETYPE_ID_MAP = {
+    "Configurable Firewall Filter": 4,
+    "Juniper Networks Firewall and VPN": 5,
+    "Cisco PIX Firewall": 6,
+    "Apache HTTP Server": 10,
+    "Linux OS": 11,
+    "Microsoft Windows Security Event Log": 12,
+    "Microsoft IIS": 13,
+    "Cisco Adaptive Security Appliance (ASA)": 41,
+    "Squid Web Proxy": 46,
+    "F5 Networks BIG-IP LTM": 49,
+    "Fortinet FortiGate Security Gateway": 73,
+    "Symantec Gateway Security (SGS) Appliance": 82,
+    "Mac OS X": 102,
+    "Blue Coat SG Appliance": 103,
+    "Nortel Switched Firewall 6000": 104,
+    "Nortel Switched Firewall 5100": 120,
+    "Imperva SecureSphere": 154,
+    "ISC BIND": 185,
+    "Microsoft ISA": 191,
+    "Cisco ACE Firewall": 194,
+    "Risk Manager Default Question": 200,
+    "Palo Alto PA Series": 206,
+    "Oracle BEA WebLogic": 239,
+    "Barracuda Spam & Virus Firewall": 278,
+    "F5 Networks BIG-IP AFM": 296,
+    "Zscaler Nss": 331,
+    "Vormetric Data Security": 340,
+    "Amazon AWS CloudTrail": 347,
+    "Microsoft DNS Debug": 384,
+    "Microsoft Office 365": 397,
+    "Microsoft Azure Platform": 413,
+    "NGINX HTTP Server": 439,
+    "Microsoft Azure Active Directory": 445,
+    "Google Cloud Platform Firewall": 455,
+    "Amazon AWS Network Firewall": 456,
+}
+
+QID_NAME_ID_MAP = {
+    "ProcessAccess": 5001829,
+    "FileCreateStreamHash": 5001834,
+    "Driver loaded": 5001843,
+    "CreateRemoteThread": 5001845,
+}
+
+LOG_SOURCE_FUNCTIONS_MAP = {
+    r"CATEGORYNAME\(category\)": AQLLogSourceMap(name="category", id_map=CATEGORYNAME_ID_MAP),
+    r"LOGSOURCETYPENAME\(devicetype\)": AQLLogSourceMap(name="devicetype", id_map=DEVICETYPE_ID_MAP),
+    r"QIDNAME\(qid\)": AQLLogSourceMap(name="qid", id_map=QID_NAME_ID_MAP),
+}

uncoder-core/app/translator/platforms/base/aql/mapping.py

@@ -74,8 +74,11 @@ def get_suitable_source_mappings(
             if log_source_signature.is_suitable(devicetype, category, qid, qideventcategory):
                 if source_mapping.fields_mapping.is_suitable(field_names):
                     suitable_source_mappings.append(source_mapping)
-            elif source_mapping.fields_mapping.is_suitable(field_names):
-                suitable_source_mappings.append(source_mapping)
+
+        if not suitable_source_mappings:
+            for source_mapping in self._source_mappings.values():
+                if source_mapping.fields_mapping.is_suitable(field_names):
+                    suitable_source_mappings.append(source_mapping)
 
         if not suitable_source_mappings:
             suitable_source_mappings = [self._source_mappings[DEFAULT_MAPPING_NAME]]

uncoder-core/app/translator/platforms/base/aql/parsers/aql.py

@@ -22,6 +22,7 @@
 from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer
 from app.translator.core.parser import PlatformQueryParser
 from app.translator.platforms.base.aql.const import NUM_VALUE_PATTERN, SINGLE_QUOTES_VALUE_PATTERN
+from app.translator.platforms.base.aql.log_source_map import LOG_SOURCE_FUNCTIONS_MAP
 from app.translator.platforms.base.aql.mapping import AQLMappings, aql_mappings
 from app.translator.platforms.base.aql.tokenizer import AQLTokenizer
 from app.translator.tools.utils import get_match_group
@@ -31,10 +32,10 @@ class AQLQueryParser(PlatformQueryParser):
     tokenizer = AQLTokenizer()
     mappings: AQLMappings = aql_mappings
 
-    log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME", "LOGSOURCETYPENAME", "CATEGORYNAME")
+    log_source_functions = ("LOGSOURCENAME", "LOGSOURCEGROUPNAME")
     log_source_function_pattern = r"\(?(?P<key>___func_name___\([a-zA-Z]+\))(?:\s+like\s+|\s+ilike\s+|\s*=\s*)'(?P<value>[%a-zA-Z\s]+)'\s*\)?\s+(?:and|or)?\s"  # noqa: E501
 
-    log_source_key_types = ("devicetype", "category", "qid", "qideventcategory")
+    log_source_key_types = ("devicetype", "category", "qid", "qideventcategory", *LOG_SOURCE_FUNCTIONS_MAP.keys())
     log_source_pattern = rf"___source_type___(?:\s+like\s+|\s+ilike\s+|\s*=\s*)(?:{SINGLE_QUOTES_VALUE_PATTERN}|{NUM_VALUE_PATTERN})(?:\s+(?:and|or)\s+|\s+)?"  # noqa: E501
     num_value_pattern = r"[0-9]+"
     multi_num_log_source_pattern = (
@@ -67,6 +68,11 @@ def __parse_multi_value_log_source(
         query = query[:pos_start] + query[pos_end:]
         return query, re.findall(pattern, value)
 
+    def __map_log_source_value(self, logsource_key: str, value: Union[str, int]) -> tuple[str, Union[int, str]]:
+        if log_source_map := LOG_SOURCE_FUNCTIONS_MAP.get(logsource_key):
+            return log_source_map.name, log_source_map.id_map.get(value, value)
+        return logsource_key, value
+
     def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], list[int]]], str]:
         log_sources = {}
 
@@ -80,6 +86,7 @@ def __parse_log_sources(self, query: str) -> tuple[dict[str, Union[list[str], li
                 num_value = get_match_group(search, group_name="num_value")
                 str_value = get_match_group(search, group_name="s_q_value")
                 value = num_value and int(num_value) or str_value
+                log_source_key, value = self.__map_log_source_value(log_source_key, value)
                 log_sources.setdefault(log_source_key, []).append(value)
                 pos_start = search.start()
                 pos_end = search.end()

uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py

@@ -96,7 +96,11 @@ def is_not_none(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:
         return f"{field} != null"
 
     def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str:  # noqa: ARG002
-        raise UnsupportedRenderMethod(platform_name=self.details.name, method="Keywords")
+        if isinstance(value, list):
+            return f"({self.or_token.join(self.contains_modifier(field=field, value=v) for v in value)})"
+        if value.endswith("\\"):
+            return f'_raw_log ~= ".*{self.apply_value(value, value_type=ValueType.regex_value)}.*"'
+        return f'_raw_log contains "{self.apply_value(value)}"'
 
 
 @render_manager.register