[Backport 8.x] Add support for synthetic_source_keep = none (elastic#…

…2422)

Add support for synthetic_source_keep mapping in generated elasticsearch component files, and add this mapping field to all ECS fields that represent sets.

synthetic_source_keep = none indicates that field is an unordered set, and helps improve storage efficiency with Elasticsearch logsdb index mode.

CHANGELOG.next.md

@@ -18,6 +18,9 @@ Thanks, you're awesome :-) -->
 
 #### Improvements
 
+* Set synthetic_source_keep = none on fields that represent sets. #2422
+
+#### Deprecated
 
 ### Tooling and Artifact Changes
 

experimental/generated/ecs/ecs_flat.yml

@@ -628,6 +628,7 @@ client.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 cloud.account.id:
   dashed_name: cloud-account-id
@@ -1131,6 +1132,7 @@ container.image.tag:
   normalize:
   - array
   short: Container image tags.
+  synthetic_source_keep: none
   type: keyword
 container.labels:
   dashed_name: container-labels
@@ -1708,6 +1710,7 @@ destination.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 device.id:
   dashed_name: device-id
@@ -2412,6 +2415,7 @@ dns.header_flags:
   normalize:
   - array
   short: Array of DNS header flags.
+  synthetic_source_keep: none
   type: keyword
 dns.id:
   dashed_name: dns-id
@@ -2744,6 +2748,7 @@ email.bcc.address:
   normalize:
   - array
   short: Email address of BCC recipient
+  synthetic_source_keep: none
   type: keyword
 email.cc.address:
   dashed_name: email-cc-address
@@ -2756,6 +2761,7 @@ email.cc.address:
   normalize:
   - array
   short: Email address of CC recipient
+  synthetic_source_keep: none
   type: keyword
 email.content_type:
   dashed_name: email-content-type
@@ -2804,6 +2810,7 @@ email.from.address:
   normalize:
   - array
   short: The sender's email address.
+  synthetic_source_keep: none
   type: keyword
 email.local_id:
   dashed_name: email-local-id
@@ -2853,6 +2860,7 @@ email.reply_to.address:
   normalize:
   - array
   short: Address replies should be delivered to.
+  synthetic_source_keep: none
   type: keyword
 email.sender.address:
   dashed_name: email-sender-address
@@ -2864,6 +2872,7 @@ email.sender.address:
   name: sender.address
   normalize: []
   short: Address of the message sender.
+  synthetic_source_keep: none
   type: keyword
 email.subject:
   dashed_name: email-subject
@@ -2891,6 +2900,7 @@ email.to.address:
   normalize:
   - array
   short: Email address of recipient
+  synthetic_source_keep: none
   type: keyword
 email.x_mailer:
   dashed_name: email-x-mailer
@@ -3237,6 +3247,7 @@ event.category:
   normalize:
   - array
   short: Event category. The second categorization field in the hierarchy.
+  synthetic_source_keep: none
   type: keyword
 event.code:
   dashed_name: event-code
@@ -3798,6 +3809,7 @@ event.type:
   normalize:
   - array
   short: Event type. The third categorization field in the hierarchy.
+  synthetic_source_keep: none
   type: keyword
 event.url:
   dashed_name: event-url
@@ -3925,6 +3937,7 @@ file.attributes:
   normalize:
   - array
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 file.code_signature.digest_algorithm:
   dashed_name: file-code-signature-digest-algorithm
@@ -5917,6 +5930,7 @@ host.ip:
   normalize:
   - array
   short: Host ip addresses.
+  synthetic_source_keep: none
   type: ip
 host.mac:
   dashed_name: host-mac
@@ -5934,6 +5948,7 @@ host.mac:
   - array
   pattern: ^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$
   short: Host MAC addresses.
+  synthetic_source_keep: none
   type: keyword
 host.name:
   dashed_name: host-name
@@ -7194,6 +7209,7 @@ observer.ip:
   normalize:
   - array
   short: IP addresses of the observer.
+  synthetic_source_keep: none
   type: ip
 observer.mac:
   dashed_name: observer-mac
@@ -7211,6 +7227,7 @@ observer.mac:
   - array
   pattern: ^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$
   short: MAC addresses of the observer.
+  synthetic_source_keep: none
   type: keyword
 observer.name:
   dashed_name: observer-name
@@ -7473,6 +7490,7 @@ orchestrator.resource.annotation:
   normalize:
   - array
   short: The list of annotations added to the resource.
+  synthetic_source_keep: none
   type: keyword
 orchestrator.resource.id:
   dashed_name: orchestrator-resource-id
@@ -7495,6 +7513,7 @@ orchestrator.resource.ip:
   normalize:
   - array
   short: IP address assigned to the resource associated with the event being observed.
+  synthetic_source_keep: none
   type: ip
 orchestrator.resource.label:
   dashed_name: orchestrator-resource-label
@@ -7507,6 +7526,7 @@ orchestrator.resource.label:
   normalize:
   - array
   short: The list of labels added to the resource.
+  synthetic_source_keep: none
   type: keyword
 orchestrator.resource.name:
   dashed_name: orchestrator-resource-name
@@ -8996,6 +9016,7 @@ process.env_vars:
   normalize:
   - array
   short: Array of environment variable bindings.
+  synthetic_source_keep: none
   type: keyword
 process.executable:
   dashed_name: process-executable
@@ -11452,6 +11473,7 @@ process.parent.thread.capabilities.effective:
   original_fieldset: process
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities used for permission checks.
+  synthetic_source_keep: none
   type: keyword
 process.parent.thread.capabilities.permitted:
   dashed_name: process-parent-thread-capabilities-permitted
@@ -11467,6 +11489,7 @@ process.parent.thread.capabilities.permitted:
   original_fieldset: process
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities a thread could assume.
+  synthetic_source_keep: none
   type: keyword
 process.parent.thread.id:
   dashed_name: process-parent-thread-id
@@ -12660,6 +12683,7 @@ process.thread.capabilities.effective:
   - array
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities used for permission checks.
+  synthetic_source_keep: none
   type: keyword
 process.thread.capabilities.permitted:
   dashed_name: process-thread-capabilities-permitted
@@ -12674,6 +12698,7 @@ process.thread.capabilities.permitted:
   - array
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities a thread could assume.
+  synthetic_source_keep: none
   type: keyword
 process.thread.id:
   dashed_name: process-thread-id
@@ -12944,6 +12969,7 @@ related.hash:
   normalize:
   - array
   short: All the hashes seen on your event.
+  synthetic_source_keep: none
   type: keyword
 related.hosts:
   dashed_name: related-hosts
@@ -12956,6 +12982,7 @@ related.hosts:
   normalize:
   - array
   short: All the host identifiers seen on your event.
+  synthetic_source_keep: none
   type: keyword
 related.ip:
   dashed_name: related-ip
@@ -12966,6 +12993,7 @@ related.ip:
   normalize:
   - array
   short: All of the IPs seen on your event.
+  synthetic_source_keep: none
   type: ip
 related.user:
   dashed_name: related-user
@@ -12977,6 +13005,7 @@ related.user:
   normalize:
   - array
   short: All the user names or other user identifiers seen on the event.
+  synthetic_source_keep: none
   type: keyword
 rule.author:
   dashed_name: rule-author
@@ -12990,6 +13019,7 @@ rule.author:
   normalize:
   - array
   short: Rule author
+  synthetic_source_keep: none
   type: keyword
 rule.category:
   dashed_name: rule-category
@@ -13560,6 +13590,7 @@ server.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 service.address:
   dashed_name: service-address
@@ -13708,6 +13739,7 @@ service.node.roles:
   normalize:
   - array
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.origin.address:
   dashed_name: service-origin-address
@@ -13864,6 +13896,7 @@ service.origin.node.roles:
   - array
   original_fieldset: service
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.origin.state:
   dashed_name: service-origin-state
@@ -14073,6 +14106,7 @@ service.target.node.roles:
   - array
   original_fieldset: service
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.target.state:
   dashed_name: service-target-state
@@ -14607,6 +14641,7 @@ source.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 span.id:
   dashed_name: span-id
@@ -14633,6 +14668,7 @@ tags:
   normalize:
   - array
   short: List of keywords used to tag each event.
+  synthetic_source_keep: none
   type: keyword
 threat.enrichments:
   dashed_name: threat-enrichments
@@ -14644,6 +14680,7 @@ threat.enrichments:
   normalize:
   - array
   short: List of objects containing indicators enriching the event.
+  synthetic_source_keep: none
   type: nested
 threat.enrichments.indicator:
   dashed_name: threat-enrichments-indicator
@@ -14752,6 +14789,7 @@ threat.enrichments.indicator.file.attributes:
   - array
   original_fieldset: file
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 threat.enrichments.indicator.file.code_signature.digest_algorithm:
   dashed_name: threat-enrichments-indicator-file-code-signature-digest-algorithm
@@ -17349,6 +17387,7 @@ threat.group.alias:
   normalize:
   - array
   short: Alias of the group.
+  synthetic_source_keep: none
   type: keyword
 threat.group.id:
   dashed_name: threat-group-id
@@ -17487,6 +17526,7 @@ threat.indicator.file.attributes:
   - array
   original_fieldset: file
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 threat.indicator.file.code_signature.digest_algorithm:
   dashed_name: threat-indicator-file-code-signature-digest-algorithm
@@ -19184,6 +19224,7 @@ threat.indicator.id:
   normalize:
   - array
   short: ID of the indicator
+  synthetic_source_keep: none
   type: keyword
 threat.indicator.ip:
   dashed_name: threat-indicator-ip
@@ -19973,6 +20014,7 @@ threat.software.alias:
   normalize:
   - array
   short: Alias of the software
+  synthetic_source_keep: none
   type: keyword
 threat.software.id:
   dashed_name: threat-software-id
@@ -20024,6 +20066,7 @@ threat.software.platforms:
   normalize:
   - array
   short: Platforms of the software.
+  synthetic_source_keep: none
   type: keyword
 threat.software.reference:
   dashed_name: threat-software-reference
@@ -21510,6 +21553,7 @@ user.changes.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.domain:
   dashed_name: user-domain
@@ -21653,6 +21697,7 @@ user.effective.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.email:
   dashed_name: user-email
@@ -21845,6 +21890,7 @@ user.roles:
   normalize:
   - array
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.target.domain:
   dashed_name: user-target-domain
@@ -21976,6 +22022,7 @@ user.target.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user_agent.device.name:
   dashed_name: user-agent-device-name
@@ -22335,6 +22382,7 @@ vulnerability.category:
   normalize:
   - array
   short: Category of a vulnerability.
+  synthetic_source_keep: none
   type: keyword
 vulnerability.classification:
   dashed_name: vulnerability-classification