diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py new file mode 100644 index 00000000..a8788ada --- /dev/null +++ b/uncoder-core/app/translator/core/const.py @@ -0,0 +1,6 @@ +from typing import Union + +from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.identifier import Identifier + +TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py index 591883d8..8ff6ccfb 100644 --- a/uncoder-core/app/translator/core/context_vars.py +++ b/uncoder-core/app/translator/core/context_vars.py @@ -1,4 +1,10 @@ from contextvars import ContextVar +from typing import Optional return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) """Set to True to return only first query if rendered multiple options""" + +wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True) +"""Set to False not to wrap query with meta info commentary""" + +preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None) diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py index bb31e77f..8e609a03 100644 --- a/uncoder-core/app/translator/core/custom_types/functions.py +++ b/uncoder-core/app/translator/core/custom_types/functions.py @@ -28,6 +28,7 @@ class FunctionType(CustomEnum): bin = "bin" eval = "eval" fields = "fields" + join = "join" rename = "rename" search = "search" sort_limit = "sort_limit" diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 2576f93a..95556dcc 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma self.__generic_names_map = generic_names_map +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) + self.alias_right = Alias(name=source_name_right) if is_alias_right else None + + class FieldValue: def __init__( self, diff --git a/uncoder-core/app/translator/core/models/functions/join.py b/uncoder-core/app/translator/core/models/functions/join.py new file mode 100644 index 00000000..0f44da68 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/join.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass, field +from typing import Union + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.field import Alias, Field +from app.translator.core.models.functions.base import Function +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.tools.custom_enum import CustomEnum + + +class JoinType(CustomEnum): + inner = "inner" + left = "left" + right = "right" + cross = "cross" + + +@dataclass +class JoinFunction(Function): + name: str = FunctionType.join + alias: Alias = None + type_: str = JoinType.inner + tokenized_query_container: TokenizedQueryContainer = None + condition: list[Union[Alias, Field, Identifier]] = field(default_factory=list) + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/functions/union.py b/uncoder-core/app/translator/core/models/functions/union.py new file mode 100644 index 00000000..ef4d4179 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/union.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_container import TokenizedQueryContainer + + +@dataclass +class UnionFunction(Function): + name: str = FunctionType.union + tokenized_query_container: TokenizedQueryContainer = None + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index dccfc180..0d90f237 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 7cc10ec1..18b50739 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping @@ -28,7 +29,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer +from app.translator.core.tokenizer import QueryTokenizer class QueryParser(ABC): diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index b326cf96..b66f4430 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ - +import itertools from abc import ABC, abstractmethod from collections.abc import Callable from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import TOKEN_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -30,22 +31,21 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValue, StrValueManager -from app.translator.core.tokenizer import TOKEN_TYPE -class BaseQueryFieldValue(ABC): +class BaseFieldValueRender(ABC): details: PlatformDetails = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None def __init__(self, or_token: str): - self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { + self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { OperatorType.EQ: self.equal_modifier, OperatorType.NOT_EQ: self.not_equal_modifier, OperatorType.LT: self.less_modifier, @@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) return self.escape_manager.escape(value, value_type) def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str: - if modifier_function := self.field_value.get(operator.token_type): + if modifier_function := self.modifiers_map.get(operator.token_type): return modifier_function(field, value) raise UnsupportedOperatorException(operator.token_type) +class BaseFieldFieldRender(ABC): + operators_map: ClassVar[dict[str, str]] = {} + + def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str: + if mapped_operator := self.operators_map.get(operator.token_type): + return f"{field_left} {mapped_operator} {field_right}" + raise UnsupportedOperatorException(operator.token_type) + + class QueryRender(ABC): comment_symbol: str = None details: PlatformDetails = None @@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str: not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions) return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}") + def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str: + if not_supported_functions and wrap_query_with_meta_info_ctx_var.get(): + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return query + rendered_not_supported + + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender): group_token = "(%s)" query_parts_delimiter = " " - field_value_map = BaseQueryFieldValue(or_token=or_token) + field_field_render = BaseFieldFieldRender() + field_value_render = BaseFieldValueRender(or_token=or_token) raw_log_field_pattern_map: ClassVar[dict[str, str]] = None def __init__(self): super().__init__() - self.operator_map = { + self.logical_operators_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", @@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): - if token.alias: - field_name = token.alias.name - else: - mapped_fields = self.map_field(token.field, source_mapping) - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value( - field=field, operator=token.operator, value=token.value - ) - for field in mapped_fields - ] - ) - - field_name = mapped_fields[0] - - return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value) - + mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + if isinstance(token, FieldField): + alias_left, field_left = token.alias_left, token.field_left + mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + alias_right, field_right = token.alias_right, token.field_right + mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_field_render.apply_field_field(pair[0], token.operator, pair[1]) + for pair in cross_paired_fields + ] + ) + return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) if isinstance(token, Keyword): - return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value) + return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: - return self.operator_map.get(token.token_type) + return self.logical_operators_map.get(token.token_type) return token.token_type @@ -273,8 +293,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str: - if meta_info and (meta_info.id or meta_info.title): + def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: + if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, @@ -307,11 +327,8 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]: @@ -342,7 +359,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: return self.finalize_query( prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) @@ -380,7 +397,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -417,6 +434,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 45486ef1..7530af5f 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,18 +30,18 @@ UnsupportedOperatorException, ) from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction +from app.translator.core.models.functions.join import JoinFunction from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg +from app.translator.core.models.functions.union import UnionFunction from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field] - class BaseTokenizer(ABC): @abstractmethod @@ -323,13 +324,18 @@ def filter_tokens( ) -> list[TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] - def get_field_tokens_from_func_args( + def get_field_tokens_from_func_args( # noqa: PLR0912 self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]] ) -> list[Field]: result = [] for arg in args: if isinstance(arg, Field): result.append(arg) + elif isinstance(arg, FieldField): + if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name: + result.append(arg.field_left) + if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name: + result.append(arg.field_right) elif isinstance(arg, FieldValue): if not arg.alias or arg.alias.name != arg.field.source_name: result.append(arg.field) @@ -337,6 +343,8 @@ def get_field_tokens_from_func_args( result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) + elif isinstance(arg, (JoinFunction, UnionFunction)): + continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a62e5b00..8550c94a 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -21,10 +21,10 @@ from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -class AthenaFieldValue(SqlFieldValue): +class AthenaFieldValueRender(SqlFieldValueRender): details: PlatformDetails = athena_details @@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = AthenaFieldValue(or_token=or_token) + field_value_render = AthenaFieldValueRender(or_token=or_token) comment_symbol = "--" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 05826d08..6c0c1665 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager -class AQLFieldValue(BaseQueryFieldValue): +class AQLFieldValueRender(BaseFieldValueRender): str_value_manager = aql_str_value_manager @staticmethod @@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = AQLFieldValue(or_token=or_token) - def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) extra_condition = log_source_signature.extra_condition diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index b5994499..f8511d82 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager -class LuceneFieldValue(BaseQueryFieldValue): +class LuceneFieldValueRender(BaseFieldValueRender): str_value_manager = lucene_str_value_manager @staticmethod diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index b2c12068..74adf32b 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -21,11 +21,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager -class SplFieldValue(BaseQueryFieldValue): +class SplFieldValueRender(BaseFieldValueRender): escape_manager = spl_escape_manager def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 43904a1e..d69f1590 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -22,10 +22,10 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -class SqlFieldValue(BaseQueryFieldValue): +class SqlFieldValueRender(BaseFieldValueRender): def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 4101b825..8bcbe56f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -23,14 +23,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings -class ChronicleFieldValue(BaseQueryFieldValue): +class ChronicleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = chronicle_query_details escape_manager = chronicle_escape_manager @@ -109,6 +109,6 @@ class ChronicleQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = ChronicleFieldValue(or_token=or_token) + field_value_render = ChronicleFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index aaa64384..1961c72b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,12 +26,12 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details -from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender +from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." -class ChronicleRuleFieldValue(ChronicleFieldValue): +class ChronicleRuleFieldValueRender(ChronicleFieldValueRender): details: PlatformDetails = chronicle_rule_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -85,7 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details or_token = "or" - field_value_map = ChronicleRuleFieldValue(or_token=or_token) + field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @staticmethod def prepare_title(title: str) -> str: diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 8c6630e9..3e5900cc 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeFieldValue(SplFieldValue): +class CrowdStrikeFieldValueRender(SplFieldValueRender): details = crowdstrike_query_details @@ -36,7 +36,7 @@ class CrowdStrikeQueryRender(SplQueryRender): platform_functions: CrowdStrikeFunctions = None or_token = "OR" - field_value_map = CrowdStrikeFieldValue(or_token=or_token) + field_value_render = CrowdStrikeFieldValueRender(or_token=or_token) comment_symbol = "`" def init_platform_functions(self) -> None: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 09fad79b..0b7b20c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -50,7 +50,7 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) + field_value_render = ElasticSearchRuleFieldValue(or_token=or_token) def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -109,7 +109,4 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 104b8ecc..9d7914ab 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -49,7 +49,7 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) + field_value_render = ElasticAlertRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -75,7 +75,4 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 8d2db1d0..2e6a12f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchFieldValue(LuceneFieldValue): +class ElasticSearchFieldValue(LuceneFieldValueRender): details: PlatformDetails = elasticsearch_lucene_query_details @@ -34,4 +34,4 @@ class ElasticSearchQueryRender(LuceneQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = ElasticSearchFieldValue(or_token=or_token) + field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index c3b6a46a..53a4acf5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -45,7 +45,7 @@ class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = KibanaFieldValue(or_token=or_token) + field_value_render = KibanaFieldValue(or_token=or_token) def finalize_query( self, @@ -74,7 +74,4 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 9a013dcf..d8421977 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -45,7 +45,7 @@ class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = XpackWatcherRuleFieldValue(or_token=or_token) + field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -78,7 +78,4 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 65ca0b07..dfbc2ee6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -28,9 +29,8 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.core.tokenizer import TOKEN_TYPE from app.translator.managers import render_manager from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -76,7 +76,7 @@ ] -class FortiSiemFieldValue(BaseQueryFieldValue): +class FortiSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = forti_siem_rule_details str_value_manager = forti_siem_str_value_manager @@ -194,7 +194,7 @@ class FortiSiemRuleRender(PlatformQueryRender): group_token = "(%s)" - field_value_map = FortiSiemFieldValue(or_token=or_token) + field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: @@ -244,7 +244,7 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -324,11 +324,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) - - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod def get_attr_str(fields: set[str]) -> str: diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 2bdf001e..986ddd93 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogFieldValue(LuceneFieldValue): +class GraylogFieldValue(LuceneFieldValueRender): details: PlatformDetails = graylog_details @@ -34,4 +34,4 @@ class GraylogQueryRender(LuceneQueryRender): mappings: GraylogMappings = graylog_mappings or_token = "OR" - field_value_map = GraylogFieldValue(or_token=or_token) + field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 0348bfb0..3c73c234 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender from app.translator.platforms.hunters.const import hunters_details from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings -class HuntersFieldValue(SqlFieldValue): +class HuntersFieldValueRender(SqlFieldValueRender): details: PlatformDetails = hunters_details @@ -35,7 +35,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = HuntersFieldValue(or_token=or_token) + field_value_render = HuntersFieldValueRender(or_token=or_token) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 624fa3d7..4a288491 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager @@ -41,7 +41,7 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmAxonFieldValue(BaseQueryFieldValue): +class LogRhythmAxonFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_axon_query_details escape_manager = logrhythm_query_escape_manager @@ -204,7 +204,7 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmAxonFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" @@ -224,7 +224,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -232,20 +232,17 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 20514140..2e68c2d1 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -29,7 +29,7 @@ from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( - LogRhythmAxonFieldValue, + LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -44,7 +44,7 @@ } -class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): +class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): details: PlatformDetails = logrhythm_axon_rule_details escape_manager = logrhythm_rule_escape_manager @@ -53,7 +53,7 @@ class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details or_token = "or" - field_value_map = LogRhythmAxonRuleFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) def finalize_query( self, @@ -93,7 +93,4 @@ def finalize_query( ] json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index 9cb7cf05..e1ed4818 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -23,7 +23,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager @@ -31,7 +31,7 @@ from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings -class LogScaleFieldValue(BaseQueryFieldValue): +class LogScaleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logscale_query_details escape_manager = logscale_escape_manager @@ -102,7 +102,7 @@ class LogScaleQueryRender(PlatformQueryRender): and_token = "" not_token = "not" - field_value_map = LogScaleFieldValue(or_token=or_token) + field_value_render = LogScaleFieldValueRender(or_token=or_token) def init_platform_functions(self) -> None: self.platform_functions = log_scale_functions @@ -123,8 +123,5 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 4b3af0fb..a6628045 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,13 +26,13 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details -from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender +from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Falcon LogScale Alert" -class LogScaleAlertFieldValue(LogScaleFieldValue): +class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): details: PlatformDetails = logscale_alert_details @@ -40,7 +40,7 @@ class LogScaleAlertFieldValue(LogScaleFieldValue): class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details or_token = "or" - field_value_map = LogScaleAlertFieldValue(or_token=or_token) + field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) def finalize_query( self, @@ -70,8 +70,5 @@ def finalize_query( mitre_attack=mitre_attack, ) - json_query = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_query + rendered_not_supported - return json_query + rule_str = json.dumps(rule, indent=4, sort_keys=False) + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 7b7a3779..38617b55 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -23,12 +23,12 @@ from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) -class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_defender_details @@ -38,7 +38,7 @@ class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = None or_token = "or" - field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) + field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) is_strict_mapping = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 3153f8d4..7ef6f1f9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -22,7 +22,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -30,7 +30,7 @@ from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings -class MicrosoftSentinelFieldValue(BaseQueryFieldValue): +class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): details: PlatformDetails = microsoft_sentinel_query_details escape_manager = microsoft_escape_manager @@ -128,7 +128,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e2fdb81f..b5631ef5 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -42,7 +42,7 @@ } -class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_sentinel_rule_details @@ -50,7 +50,7 @@ class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details or_token = "or" - field_value_map = MicrosoftSentinelRuleFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) def __create_mitre_threat(self, meta_info: MetaInfoContainer) -> tuple[list, list]: tactics = set() @@ -92,7 +92,4 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 1d2145a7..3298c106 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,12 +24,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchFieldValue(LuceneFieldValue): +class OpenSearchFieldValueRender(LuceneFieldValueRender): details: PlatformDetails = opensearch_query_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -102,4 +102,4 @@ class OpenSearchQueryRender(LuceneQueryRender): mappings: OpenSearchMappings = opensearch_mappings or_token = "OR" - field_value_map = OpenSearchFieldValue(or_token=or_token) + field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 3f68e6c6..106a1fe1 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -30,13 +30,13 @@ from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender +from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" _SEVERITIES_MAP = {SeverityType.critical: "5", SeverityType.high: "4", SeverityType.medium: "3", SeverityType.low: "2"} -class OpenSearchRuleFieldValue(OpenSearchFieldValue): +class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): details: PlatformDetails = opensearch_rule_details @@ -49,7 +49,7 @@ class OpenSearchRuleRender(OpenSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = OpenSearchRuleFieldValue(or_token=or_token) + field_value_render = OpenSearchRuleFieldValueRender(or_token=or_token) def __init__(self): super().__init__() @@ -76,10 +76,7 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index ba673cd2..bc57adee 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -20,12 +20,14 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.mapping import SourceMapping from app.translator.core.models.field import FieldValue, Keyword from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.palo_alto.const import cortex_xql_query_details @@ -48,7 +50,8 @@ } -class CortexXQLFieldValue(BaseQueryFieldValue): + +class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details str_value_manager = cortex_xql_str_value_manager @@ -145,6 +148,17 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"_raw_log contains {self._pre_process_value(field ,value, value_type=ValueType.value, wrap_str=True)}" +class CortexXQLFieldFieldRender(BaseFieldFieldRender): + operators_map: ClassVar[dict[str, str]] = { + OperatorType.EQ: "=", + OperatorType.NOT_EQ: "!=", + OperatorType.LT: "<", + OperatorType.LTE: "<=", + OperatorType.GT: ">", + OperatorType.GTE: ">=", + } + + @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details @@ -162,7 +176,8 @@ class CortexXQLQueryRender(PlatformQueryRender): not_token = "not" query_parts_delimiter = "\n" - field_value_map = CortexXQLFieldValue(or_token=or_token) + field_field_render = CortexXQLFieldFieldRender() + field_value_render = CortexXQLFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = False @@ -184,7 +199,8 @@ def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" - return f"{functions_prefix}{log_source_signature}" + log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) + return f"{functions_prefix}{log_source_str}" def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index 0f06fb40..cf4a7d51 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,14 +19,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender +from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details -class QradarFieldValue(AQLFieldValue): +class QradarFieldValueRender(AQLFieldValueRender): details: PlatformDetails = qradar_query_details @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index dc33a507..856fd4a3 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -281,10 +281,10 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: raise NotImplementedError - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() meta_info = query_container.meta_info @@ -316,6 +316,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index f9404cac..e14c6bfc 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkFieldValue(SplFieldValue): +class SplunkFieldValueRender(SplFieldValueRender): details: PlatformDetails = splunk_query_details @@ -35,7 +35,7 @@ class SplunkQueryRender(SplQueryRender): or_token = "OR" - field_value_map = SplunkFieldValue(or_token=or_token) + field_value_render = SplunkFieldValueRender(or_token=or_token) mappings: SplunkMappings = splunk_mappings platform_functions: SplunkFunctions = None diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index ef0d097d..5dc2096a 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,14 +25,14 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details -from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender +from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Splunk Alert" _SEVERITIES_MAP = {SeverityType.critical: "4", SeverityType.high: "3", SeverityType.medium: "2", SeverityType.low: "1"} -class SplunkAlertFieldValue(SplunkFieldValue): +class SplunkAlertFieldValueRender(SplunkFieldValueRender): details: PlatformDetails = splunk_alert_details @@ -40,7 +40,7 @@ class SplunkAlertFieldValue(SplunkFieldValue): class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details or_token = "OR" - field_value_map = SplunkAlertFieldValue(or_token=or_token) + field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @staticmethod def __create_mitre_threat(meta_info: MetaInfoContainer) -> dict: @@ -74,7 +74,4 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions)