diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 415fb19aeac..8987791bf8a 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.39.0" + changes: + - description: Add `related.entity` field to audit logs. + type: enhancement + link: https://github.com/elastic/integrations/pull/11762 - version: "2.38.0" changes: - description: Add `policy_violation_info`, `metadata` and `related` fields to audit logs. diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json index 8ea7905c637..dc2cb2a31f5 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json @@ -59,6 +59,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats", + "xxx@xxx.xxx" ] }, "service": { @@ -139,6 +143,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/machineTypes", + "xxx@xxx.xxx" ] }, "service": { @@ -244,6 +252,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/instances", + "xxx@xxx.xxx" ] }, "service": { @@ -336,6 +348,10 @@ ], "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/elastic-beats/global/instances", + "xxx@xxx.xxx" ] }, "service": { @@ -475,7 +491,8 @@ ], "user": [ "system:serviceaccount:cert-manager:cert-manager-webhook" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -598,6 +615,10 @@ ], "user": [ "user@mycompany.com" + ], + "entity": [ + "projects/foo/global/images/windows-server-2016-v20200805", + "user@mycompany.com" ] }, "service": { @@ -689,6 +710,10 @@ ], "user": [ "user@mycompany.com" + ], + "entity": [ + "projects/foo/zones/us-central1-a/instances/win10-test", + "user@mycompany.com" ] }, "service": { @@ -792,7 +817,8 @@ ], "user": [ "xxx@xxx.xxx" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -880,7 +906,8 @@ ], "user": [ "xxx@xxx.xxx" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -965,7 +992,8 @@ ], "user": [ "system:anonymous" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -1048,7 +1076,8 @@ ], "user": [ "system:serviceaccount:kube-system:generic-garbage-collector" - ] + ], + "entity": [] }, "service": { "name": "k8s.io" @@ -1131,6 +1160,12 @@ "related": { "user": [ "xxx@xxx.xxx" + ], + "entity": [ + "projects/project", + "sub", + "xxx@xxx.xxx", + "//xxx@xxx" ] }, "service": { @@ -1266,6 +1301,7 @@ "type": "kubernetes" }, "related": { + "entity": [], "ip": [ "67.43.156.13" ], @@ -1656,6 +1692,7 @@ "type": "kubernetes" }, "related": { + "entity": [], "ip": [ "10.142.0.152" ], @@ -1747,6 +1784,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:service-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -1826,6 +1866,10 @@ "logger": "projects/elastic/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "projects/_/buckets/dataflow-staging-us-central1-xxx/objects/staging/jfxrt-xxx.jar", + "xxx-compute@developer.gserviceaccount.com" + ], "user": [ "xxx-compute@developer.gserviceaccount.com" ] @@ -1909,6 +1953,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -1992,6 +2039,12 @@ "logger": "projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "projects/project", + "sub", + "xxx@xxx.xxx", + "//xxx@xxx" + ], "user": [ "xxx@xxx.xxx" ] @@ -2060,6 +2113,10 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fsystem_event" }, "related": { + "entity": [ + "projects/elastic-siem/zones/us-central1-c/instances/sep-perf-debian-11-155", + "system@google.com" + ], "user": [ "system@google.com" ] @@ -2138,6 +2195,9 @@ "logger": "projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fpolicy" }, "related": { + "entity": [ + "projects/elastic-siem" + ], "ip": [ "192.168.1.1" ] @@ -2236,6 +2296,9 @@ "type": "kubernetes" }, "related": { + "entity": [ + "serviceAccount:servoce-xxxx@developer.gserviceaccount.com" + ], "ip": [ "192.168.1.1" ], @@ -2311,6 +2374,9 @@ }, "type": "kubernetes" }, + "related": { + "entity": [] + }, "service": { "name": "container.googleapis.com" }, @@ -2319,4 +2385,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log new file mode 100644 index 00000000000..aef09eca3f0 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"-30102re2sad8","logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"made-up-ci-account@project-id.iam.gserviceaccount.com","principalSubject":"serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com","serviceAccountDelegationInfo":[{"principalSubject":"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..."}]},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.setIamPolicy","permissionType":"ADMIN_WRITE","resource":"projects/project-id","resourceAttributes":{"name":"projects/project-id","service":"cloudresourcemanager.googleapis.com","type":"cloudresourcemanager.googleapis.com/Project"}},{"granted":true,"permission":"resourcemanager.projects.setIamPolicy","permissionType":"ADMIN_WRITE","resource":"projects/project-id","resourceAttributes":{"name":"projects/project-id","service":"cloudresourcemanager.googleapis.com","type":"cloudresourcemanager.googleapis.com/Project"}}],"methodName":"SetIamPolicy","request":{"@type":"type.googleapis.com/google.iam.v1.SetIamPolicyRequest","policy":{"bindings":[{"members":["serviceAccount:member-sa@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/ThatRoleToo"},{"members":["serviceAccount:a@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/x"},{"members":["serviceAccount:b@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/this_role_as_well"},{"members":["serviceAccount:c@project-id.iam.gserviceaccount.com","serviceAccount:d@project-id.iam.gserviceaccount.com","serviceAccount:e@project-id.iam.gserviceaccount.com"],"role":"roles/browser"},{"members":["serviceAccount:f@project-id.iam.gserviceaccount.com","serviceAccount:g@project-id.iam.gserviceaccount.com","serviceAccount:c@project-id.iam.gserviceaccount.com"],"role":"roles/cloudasset.viewer"},{"members":["user:doesnotexist@elastic.co"],"role":"roles/cloudkms.admin"},{"members":["group:agroup@elastic.co"],"role":"roles/owner"}],"etag":"BwYnObHBOBA="},"resource":"project-id"},"requestMetadata":{"callerIp":"192.168.0.1","callerSuppliedUserAgent":"google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/project-id","response":{"@type":"type.googleapis.com/google.iam.v1.Policy","bindings":[{"members":["serviceAccount:first@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/ThatRoleToo"},{"members":["serviceAccount:second@project-id.iam.gserviceaccount.com"],"role":"projects/project-id/roles/random"}],"etag":"BwYnQ8iRtu0="},"serviceData":{"@type":"type.googleapis.com/google.iam.v1.logging.AuditData","policyDelta":{"bindingDeltas":[{"action":"ADD","member":"serviceAccount:project-id@cloudservices.gserviceaccount.com","role":"roles/resourcemanager.projectIamAdmin"}]}},"serviceName":"cloudresourcemanager.googleapis.com","status":{}},"receiveTimestamp":"2024-11-19T13:12:21.785498724Z","resource":{"labels":{"project_id":"project-id"},"type":"project"},"severity":"NOTICE","timestamp":"2024-11-19T13:12:20.942393Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json new file mode 100644 index 00000000000..2987836a147 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-cloudresourcemanager-googleapis-com.log-expected.json @@ -0,0 +1,199 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T13:12:20.942Z", + "client": { + "user": { + "email": "made-up-ci-account@project-id.iam.gserviceaccount.com", + "id": "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com" + } + }, + "cloud": { + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "SetIamPolicy", + "id": "-30102re2sad8", + "kind": "event", + "original": "{\"insertId\":\"-30102re2sad8\",\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity\",\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"made-up-ci-account@project-id.iam.gserviceaccount.com\",\"principalSubject\":\"serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com\",\"serviceAccountDelegationInfo\":[{\"principalSubject\":\"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...\"}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"resourcemanager.projects.setIamPolicy\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id\",\"resourceAttributes\":{\"name\":\"projects/project-id\",\"service\":\"cloudresourcemanager.googleapis.com\",\"type\":\"cloudresourcemanager.googleapis.com/Project\"}},{\"granted\":true,\"permission\":\"resourcemanager.projects.setIamPolicy\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id\",\"resourceAttributes\":{\"name\":\"projects/project-id\",\"service\":\"cloudresourcemanager.googleapis.com\",\"type\":\"cloudresourcemanager.googleapis.com/Project\"}}],\"methodName\":\"SetIamPolicy\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.v1.SetIamPolicyRequest\",\"policy\":{\"bindings\":[{\"members\":[\"serviceAccount:member-sa@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/ThatRoleToo\"},{\"members\":[\"serviceAccount:a@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/x\"},{\"members\":[\"serviceAccount:b@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/this_role_as_well\"},{\"members\":[\"serviceAccount:c@project-id.iam.gserviceaccount.com\",\"serviceAccount:d@project-id.iam.gserviceaccount.com\",\"serviceAccount:e@project-id.iam.gserviceaccount.com\"],\"role\":\"roles/browser\"},{\"members\":[\"serviceAccount:f@project-id.iam.gserviceaccount.com\",\"serviceAccount:g@project-id.iam.gserviceaccount.com\",\"serviceAccount:c@project-id.iam.gserviceaccount.com\"],\"role\":\"roles/cloudasset.viewer\"},{\"members\":[\"user:doesnotexist@elastic.co\"],\"role\":\"roles/cloudkms.admin\"},{\"members\":[\"group:agroup@elastic.co\"],\"role\":\"roles/owner\"}],\"etag\":\"BwYnObHBOBA=\"},\"resource\":\"project-id\"},\"requestMetadata\":{\"callerIp\":\"192.168.0.1\",\"callerSuppliedUserAgent\":\"google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{}},\"resourceName\":\"projects/project-id\",\"response\":{\"@type\":\"type.googleapis.com/google.iam.v1.Policy\",\"bindings\":[{\"members\":[\"serviceAccount:first@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/ThatRoleToo\"},{\"members\":[\"serviceAccount:second@project-id.iam.gserviceaccount.com\"],\"role\":\"projects/project-id/roles/random\"}],\"etag\":\"BwYnQ8iRtu0=\"},\"serviceData\":{\"@type\":\"type.googleapis.com/google.iam.v1.logging.AuditData\",\"policyDelta\":{\"bindingDeltas\":[{\"action\":\"ADD\",\"member\":\"serviceAccount:project-id@cloudservices.gserviceaccount.com\",\"role\":\"roles/resourcemanager.projectIamAdmin\"}]}},\"serviceName\":\"cloudresourcemanager.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2024-11-19T13:12:21.785498724Z\",\"resource\":{\"labels\":{\"project_id\":\"project-id\"},\"type\":\"project\"},\"severity\":\"NOTICE\",\"timestamp\":\"2024-11-19T13:12:20.942393Z\"}", + "outcome": "unknown", + "provider": "activity" + }, + "gcp": { + "audit": { + "authentication_info": { + "service_account_delegation_info": [ + { + "principalSubject": "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..." + } + ] + }, + "authorization_info": [ + { + "granted": true, + "permission": "resourcemanager.projects.setIamPolicy", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id", + "resource_attributes": { + "name": "projects/project-id", + "service": "cloudresourcemanager.googleapis.com", + "type": "cloudresourcemanager.googleapis.com/Project" + } + }, + { + "granted": true, + "permission": "resourcemanager.projects.setIamPolicy", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id", + "resource_attributes": { + "name": "projects/project-id", + "service": "cloudresourcemanager.googleapis.com", + "type": "cloudresourcemanager.googleapis.com/Project" + } + } + ], + "request": { + "@type": "type.googleapis.com/google.iam.v1.SetIamPolicyRequest", + "policy": { + "bindings": [ + { + "members": [ + "serviceAccount:member-sa@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/ThatRoleToo" + }, + { + "members": [ + "serviceAccount:a@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/x" + }, + { + "members": [ + "serviceAccount:b@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/this_role_as_well" + }, + { + "members": [ + "serviceAccount:c@project-id.iam.gserviceaccount.com", + "serviceAccount:d@project-id.iam.gserviceaccount.com", + "serviceAccount:e@project-id.iam.gserviceaccount.com" + ], + "role": "roles/browser" + }, + { + "members": [ + "serviceAccount:f@project-id.iam.gserviceaccount.com", + "serviceAccount:g@project-id.iam.gserviceaccount.com", + "serviceAccount:c@project-id.iam.gserviceaccount.com" + ], + "role": "roles/cloudasset.viewer" + }, + { + "members": [ + "user:doesnotexist@elastic.co" + ], + "role": "roles/cloudkms.admin" + }, + { + "members": [ + "group:agroup@elastic.co" + ], + "role": "roles/owner" + } + ], + "etag": "BwYnObHBOBA=" + }, + "resource": "project-id" + }, + "resource_name": "projects/project-id", + "response": { + "@type": "type.googleapis.com/google.iam.v1.Policy", + "bindings": [ + { + "members": [ + "serviceAccount:first@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/ThatRoleToo" + }, + { + "members": [ + "serviceAccount:second@project-id.iam.gserviceaccount.com" + ], + "role": "projects/project-id/roles/random" + } + ], + "etag": "BwYnQ8iRtu0=" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "entity": [ + "projects/project-id/roles/x", + "roles/cloudasset.viewer", + "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...", + "serviceAccount:f@project-id.iam.gserviceaccount.com", + "serviceAccount:first@project-id.iam.gserviceaccount.com", + "projects/project-id/roles/random", + "serviceAccount:b@project-id.iam.gserviceaccount.com", + "serviceAccount:d@project-id.iam.gserviceaccount.com", + "user:doesnotexist@elastic.co", + "projects/project-id/roles/ThatRoleToo", + "serviceAccount:madeupprincipal@project-id.iam.gserviceaccount.com", + "serviceAccount:member-sa@project-id.iam.gserviceaccount.com", + "serviceAccount:e@project-id.iam.gserviceaccount.com", + "serviceAccount:a@project-id.iam.gserviceaccount.com", + "serviceAccount:second@project-id.iam.gserviceaccount.com", + "projects/project-id", + "serviceAccount:g@project-id.iam.gserviceaccount.com", + "roles/cloudkms.admin", + "made-up-ci-account@project-id.iam.gserviceaccount.com", + "serviceAccount:c@project-id.iam.gserviceaccount.com", + "projects/project-id/roles/this_role_as_well", + "roles/owner", + "group:agroup@elastic.co", + "roles/browser" + ], + "ip": [ + "192.168.0.1" + ], + "user": [ + "made-up-ci-account@project-id.iam.gserviceaccount.com" + ] + }, + "service": { + "name": "cloudresourcemanager.googleapis.com" + }, + "source": { + "ip": "192.168.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "google-cloud-sdk gcloud/501.0.0 command/gcloud.projects.add-iam-policy-binding invocation-id/e9e9e4b6f9294a7da9a2247dc101225a environment/None environment-version/None client-os/LINUX client-os-ver/5.15.0 client-pltf-arch/x86_64 interactive/False from-script/False python/3.11.4 term/ (Linux 5.15.0-1074-azure),gzip(gfe)", + "os": { + "full": "Linux 5.15.0", + "name": "Linux", + "version": "5.15.0" + } + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log new file mode 100644 index 00000000000..65d6b535648 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"-w5vrlhdm7gk","labels":{"compute.googleapis.com/root_trigger_id":"UUID"},"logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"project-id@cloudservices.gserviceaccount.com","principalSubject":"serviceAccount:project-id@cloudservices.gserviceaccount.com","serviceAccountDelegationInfo":[{"firstPartyPrincipal":{"principalEmail":"principalA@prod.google.com"}}]},"authorizationInfo":[{"granted":true,"permission":"compute.instances.create","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.disks.create","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/disks/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/disks/x-logs","service":"compute","type":"compute.disks"}},{"granted":true,"permission":"compute.subnetworks.use","permissionType":"ADMIN_WRITE","resource":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","resourceAttributes":{"name":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.subnetworks.useExternalIp","permissionType":"ADMIN_WRITE","resource":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","resourceAttributes":{"name":"projects/project-id/regions/us-central1/subnetworks/x-logs-network","service":"compute","type":"compute.subnetworks"}},{"granted":true,"permission":"compute.instances.setMetadata","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setLabels","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}},{"granted":true,"permission":"compute.instances.setServiceAccount","permissionType":"ADMIN_WRITE","resource":"projects/project-id/zones/us-central1-a/instances/x-logs","resourceAttributes":{"name":"projects/project-id/zones/us-central1-a/instances/x-logs","service":"compute","type":"compute.instances"}}],"metadata":{"usedResources":{"attachedDisks":[{"isBootDisk":true,"sourceImage":"https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115","sourceImageId":"source-image-id"}]}},"methodName":"v1.compute.instances.insert","request":{"@type":"type.googleapis.com/compute.instances.insert","disks":[{"autoDelete":true,"boot":true,"deviceName":"boot","initializeParams":{"sourceImage":"https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts"},"type":"PERSISTENT"}],"machineType":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4","name":"x-logs","networkInterfaces":[{"accessConfigs":[{"name":"External NAT","type":"ONE_TO_ONE_NAT"}],"network":"https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network"}],"serviceAccounts":[{"email":"x-logs-sa@project-id.iam.gserviceaccount.com","scopes":["https://www.googleapis.com/auth/cloud-platform","https://www.googleapis.com/auth/cloudplatformorganizations"]}]},"requestMetadata":{"callerIp":"175.16.199.45","callerSuppliedUserAgent":"Google-Deployment-Manager,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2024-11-19T13:13:13.966817Z"}},"resourceLocation":{"currentLocations":["us-central1-a"]},"resourceName":"projects/project-id/zones/us-central1-a/instances/x-logs","response":{"@type":"type.googleapis.com/operation","id":"id","insertTime":"2024-11-19T05:13:13.857-08:00","name":"operation-id","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/","startTime":"2024-11-19T05:13:13.857-08:00","status":"RUNNING","targetId":"targetId","targetLink":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs","user":"project-id@cloudservices.gserviceaccount.com","zone":"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2024-11-19T13:13:14.634438657Z","resource":{"labels":{"instance_id":"2525602744967966726","project_id":"project-id","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2024-11-19T13:13:13.176899Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json new file mode 100644 index 00000000000..2d874cc183e --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-compute-googleapis-com.log-expected.json @@ -0,0 +1,255 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T13:13:13.176Z", + "client": { + "user": { + "email": "project-id@cloudservices.gserviceaccount.com", + "id": "serviceAccount:project-id@cloudservices.gserviceaccount.com" + } + }, + "cloud": { + "instance": { + "id": "2525602744967966726" + }, + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "v1.compute.instances.insert", + "category": [ + "session" + ], + "id": "-w5vrlhdm7gk", + "kind": "event", + "original": "{\"insertId\":\"-w5vrlhdm7gk\",\"labels\":{\"compute.googleapis.com/root_trigger_id\":\"UUID\"},\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Factivity\",\"operation\":{\"first\":true,\"id\":\"operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a\",\"producer\":\"compute.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalEmail\":\"project-id@cloudservices.gserviceaccount.com\",\"principalSubject\":\"serviceAccount:project-id@cloudservices.gserviceaccount.com\",\"serviceAccountDelegationInfo\":[{\"firstPartyPrincipal\":{\"principalEmail\":\"principalA@prod.google.com\"}}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"compute.instances.create\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.disks.create\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/disks/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/disks/x-logs\",\"service\":\"compute\",\"type\":\"compute.disks\"}},{\"granted\":true,\"permission\":\"compute.subnetworks.use\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"resourceAttributes\":{\"name\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"service\":\"compute\",\"type\":\"compute.subnetworks\"}},{\"granted\":true,\"permission\":\"compute.subnetworks.useExternalIp\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"resourceAttributes\":{\"name\":\"projects/project-id/regions/us-central1/subnetworks/x-logs-network\",\"service\":\"compute\",\"type\":\"compute.subnetworks\"}},{\"granted\":true,\"permission\":\"compute.instances.setMetadata\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.instances.setLabels\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}},{\"granted\":true,\"permission\":\"compute.instances.setServiceAccount\",\"permissionType\":\"ADMIN_WRITE\",\"resource\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"resourceAttributes\":{\"name\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"service\":\"compute\",\"type\":\"compute.instances\"}}],\"metadata\":{\"usedResources\":{\"attachedDisks\":[{\"isBootDisk\":true,\"sourceImage\":\"https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115\",\"sourceImageId\":\"source-image-id\"}]}},\"methodName\":\"v1.compute.instances.insert\",\"request\":{\"@type\":\"type.googleapis.com/compute.instances.insert\",\"disks\":[{\"autoDelete\":true,\"boot\":true,\"deviceName\":\"boot\",\"initializeParams\":{\"sourceImage\":\"https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts\"},\"type\":\"PERSISTENT\"}],\"machineType\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4\",\"name\":\"x-logs\",\"networkInterfaces\":[{\"accessConfigs\":[{\"name\":\"External NAT\",\"type\":\"ONE_TO_ONE_NAT\"}],\"network\":\"https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network\"}],\"serviceAccounts\":[{\"email\":\"x-logs-sa@project-id.iam.gserviceaccount.com\",\"scopes\":[\"https://www.googleapis.com/auth/cloud-platform\",\"https://www.googleapis.com/auth/cloudplatformorganizations\"]}]},\"requestMetadata\":{\"callerIp\":\"175.16.199.45\",\"callerSuppliedUserAgent\":\"Google-Deployment-Manager,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2024-11-19T13:13:13.966817Z\"}},\"resourceLocation\":{\"currentLocations\":[\"us-central1-a\"]},\"resourceName\":\"projects/project-id/zones/us-central1-a/instances/x-logs\",\"response\":{\"@type\":\"type.googleapis.com/operation\",\"id\":\"id\",\"insertTime\":\"2024-11-19T05:13:13.857-08:00\",\"name\":\"operation-id\",\"operationType\":\"insert\",\"progress\":\"0\",\"selfLink\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id\",\"selfLinkWithId\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/\",\"startTime\":\"2024-11-19T05:13:13.857-08:00\",\"status\":\"RUNNING\",\"targetId\":\"targetId\",\"targetLink\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs\",\"user\":\"project-id@cloudservices.gserviceaccount.com\",\"zone\":\"https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a\"},\"serviceName\":\"compute.googleapis.com\"},\"receiveTimestamp\":\"2024-11-19T13:13:14.634438657Z\",\"resource\":{\"labels\":{\"instance_id\":\"2525602744967966726\",\"project_id\":\"project-id\",\"zone\":\"us-central1-a\"},\"type\":\"gce_instance\"},\"severity\":\"NOTICE\",\"timestamp\":\"2024-11-19T13:13:13.176899Z\"}", + "outcome": "unknown", + "provider": "activity", + "type": [ + "start" + ] + }, + "gcp": { + "audit": { + "authentication_info": { + "service_account_delegation_info": [ + { + "firstPartyPrincipal": { + "principalEmail": "principalA@prod.google.com" + } + } + ] + }, + "authorization_info": [ + { + "granted": true, + "permission": "compute.instances.create", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.disks.create", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/disks/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/disks/x-logs", + "service": "compute", + "type": "compute.disks" + } + }, + { + "granted": true, + "permission": "compute.subnetworks.use", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "resource_attributes": { + "name": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "service": "compute", + "type": "compute.subnetworks" + } + }, + { + "granted": true, + "permission": "compute.subnetworks.useExternalIp", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "resource_attributes": { + "name": "projects/project-id/regions/us-central1/subnetworks/x-logs-network", + "service": "compute", + "type": "compute.subnetworks" + } + }, + { + "granted": true, + "permission": "compute.instances.setMetadata", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.instances.setLabels", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + }, + { + "granted": true, + "permission": "compute.instances.setServiceAccount", + "permissionType": "ADMIN_WRITE", + "resource": "projects/project-id/zones/us-central1-a/instances/x-logs", + "resource_attributes": { + "name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "service": "compute", + "type": "compute.instances" + } + } + ], + "labels": { + "compute.googleapis.com/root_trigger_id": "UUID" + }, + "logentry_operation": { + "id": "operation-1732021993132-62743cba4d27a-d29b55ba-2cd69d6a" + }, + "metadata": { + "usedResources": { + "attachedDisks": [ + { + "isBootDisk": true, + "sourceImage": "https://www.googleapis.com/compute/v1/projects/global-project/global/images/ubuntu-minimal-2204-jammy-v20241115", + "sourceImageId": "source-image-id" + } + ] + } + }, + "request": { + "@type": "type.googleapis.com/compute.instances.insert", + "disks": [ + { + "autoDelete": true, + "boot": true, + "deviceName": "boot", + "initializeParams": { + "sourceImage": "https://www.googleapis.com/compute/v1/projects/ubuntu-os-cloud/global/images/family/ubuntu-minimal-2204-lts" + }, + "type": "PERSISTENT" + } + ], + "machineType": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/machineTypes/n2-standard-4", + "name": "x-logs", + "networkInterfaces": [ + { + "accessConfigs": [ + { + "name": "External NAT", + "type": "ONE_TO_ONE_NAT" + } + ], + "network": "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network" + } + ], + "serviceAccounts": [ + { + "email": "x-logs-sa@project-id.iam.gserviceaccount.com", + "scopes": [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/cloudplatformorganizations" + ] + } + ] + }, + "resource_location": { + "current_locations": [ + "us-central1-a" + ] + }, + "resource_name": "projects/project-id/zones/us-central1-a/instances/x-logs", + "response": { + "@type": "type.googleapis.com/operation", + "id": "id", + "insertTime": "2024-11-19T05:13:13.857-08:00", + "name": "operation-id", + "operationType": "insert", + "progress": "0", + "selfLink": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/operation-id", + "selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/operations/", + "startTime": "2024-11-19T05:13:13.857-08:00", + "status_value": "RUNNING", + "targetId": "targetId", + "targetLink": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a/instances/x-logs", + "user": "project-id@cloudservices.gserviceaccount.com", + "zone": "https://www.googleapis.com/compute/v1/projects/project-id/zones/us-central1-a" + }, + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "NOTICE", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Factivity" + }, + "related": { + "entity": [ + "https://www.googleapis.com/compute/v1/projects/project-id/global/networks/x-logs-network", + "serviceAccount:project-id@cloudservices.gserviceaccount.com", + "projects/project-id/zones/us-central1-a/instances/x-logs", + "principalA@prod.google.com", + "project-id@cloudservices.gserviceaccount.com", + "x-logs-sa@project-id.iam.gserviceaccount.com" + ], + "ip": [ + "175.16.199.45" + ], + "user": [ + "project-id@cloudservices.gserviceaccount.com" + ] + }, + "service": { + "name": "compute.googleapis.com" + }, + "source": { + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.45" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Other", + "original": "Google-Deployment-Manager,gzip(gfe)" + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log new file mode 100644 index 00000000000..976a77cef60 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log @@ -0,0 +1 @@ +{"insertId":"15djrryd6bap","logName":"projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"","last":true,"producer":"iamcredentials.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalSubject":"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...","serviceAccountDelegationInfo":[{}]},"authorizationInfo":[{"granted":true,"permission":"iam.serviceAccounts.getAccessToken","permissionType":"ADMIN_READ","resourceAttributes":{}}],"metadata":{"identityDelegationChain":["projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com"]},"methodName":"GenerateAccessToken","request":{"@type":"type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest","name":"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com"},"requestMetadata":{"callerIp":"175.16.199.45","callerSuppliedUserAgent":"Go-http-client/2.0,gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2024-11-19T00:49:55.301834867Z"}},"resourceName":"projects/-/serviceAccounts/somenumber","serviceName":"iamcredentials.googleapis.com","status":{}},"receiveTimestamp":"2024-11-19T00:49:56.551702143Z","resource":{"labels":{"email_id":"made-up-ci-account@project-id.iam.gserviceaccount.com","project_id":"project-id","unique_id":"somenumber"},"type":"service_account"},"severity":"INFO","timestamp":"2024-11-19T00:49:55.293368631Z"} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json new file mode 100644 index 00000000000..3c1d622e611 --- /dev/null +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-iamcredentials-googleapis-com.log-expected.json @@ -0,0 +1,105 @@ +{ + "expected": [ + { + "@timestamp": "2024-11-19T00:49:55.293Z", + "client": { + "user": { + "id": "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/..." + } + }, + "cloud": { + "project": { + "id": "project-id" + }, + "provider": "gcp" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "GenerateAccessToken", + "category": [ + "network", + "configuration" + ], + "id": "15djrryd6bap", + "kind": "event", + "original": "{\"insertId\":\"15djrryd6bap\",\"logName\":\"projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access\",\"operation\":{\"first\":true,\"id\":\"\",\"last\":true,\"producer\":\"iamcredentials.googleapis.com\"},\"protoPayload\":{\"@type\":\"type.googleapis.com/google.cloud.audit.AuditLog\",\"authenticationInfo\":{\"principalSubject\":\"principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...\",\"serviceAccountDelegationInfo\":[{}]},\"authorizationInfo\":[{\"granted\":true,\"permission\":\"iam.serviceAccounts.getAccessToken\",\"permissionType\":\"ADMIN_READ\",\"resourceAttributes\":{}}],\"metadata\":{\"identityDelegationChain\":[\"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com\"]},\"methodName\":\"GenerateAccessToken\",\"request\":{\"@type\":\"type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest\",\"name\":\"projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com\"},\"requestMetadata\":{\"callerIp\":\"175.16.199.45\",\"callerSuppliedUserAgent\":\"Go-http-client/2.0,gzip(gfe)\",\"destinationAttributes\":{},\"requestAttributes\":{\"auth\":{},\"time\":\"2024-11-19T00:49:55.301834867Z\"}},\"resourceName\":\"projects/-/serviceAccounts/somenumber\",\"serviceName\":\"iamcredentials.googleapis.com\",\"status\":{}},\"receiveTimestamp\":\"2024-11-19T00:49:56.551702143Z\",\"resource\":{\"labels\":{\"email_id\":\"made-up-ci-account@project-id.iam.gserviceaccount.com\",\"project_id\":\"project-id\",\"unique_id\":\"somenumber\"},\"type\":\"service_account\"},\"severity\":\"INFO\",\"timestamp\":\"2024-11-19T00:49:55.293368631Z\"}", + "outcome": "success", + "provider": "data_access", + "type": [ + "access", + "allowed" + ] + }, + "gcp": { + "audit": { + "authorization_info": [ + { + "granted": true, + "permission": "iam.serviceAccounts.getAccessToken", + "permissionType": "ADMIN_READ" + } + ], + "logentry_operation": { + "id": "" + }, + "metadata": { + "identityDelegationChain": [ + "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com" + ] + }, + "request": { + "@type": "type.googleapis.com/google.iam.credentials.v1.GenerateAccessTokenRequest", + "name": "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com" + }, + "resource_name": "projects/-/serviceAccounts/somenumber", + "type": "type.googleapis.com/google.cloud.audit.AuditLog" + } + }, + "log": { + "level": "INFO", + "logger": "projects/project-id/logs/cloudaudit.googleapis.com%2Fdata_access" + }, + "related": { + "entity": [ + "projects/-/serviceAccounts/made-up-ci-account@project-id.iam.gserviceaccount.com", + "principal://iam.googleapis.com/projects/project-id/locations/global/workloadIdentityPools/...", + "projects/-/serviceAccounts/somenumber" + ], + "ip": [ + "175.16.199.45" + ] + }, + "service": { + "name": "iamcredentials.googleapis.com" + }, + "source": { + "geo": { + "city_name": "Changchun", + "continent_name": "Asia", + "country_iso_code": "CN", + "country_name": "China", + "location": { + "lat": 43.88, + "lon": 125.3228 + }, + "region_iso_code": "CN-22", + "region_name": "Jilin Sheng" + }, + "ip": "175.16.199.45" + }, + "tags": [ + "preserve_original_event" + ], + "user_agent": { + "device": { + "name": "Other" + }, + "name": "Go-http-client", + "original": "Go-http-client/2.0,gzip(gfe)", + "version": "2.0" + } + } + ] +} diff --git a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json index fb79f53d17a..1e08be1feda 100644 --- a/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json +++ b/packages/gcp/data_stream/audit/_dev/test/pipeline/test-sdh-3695.log-expected.json @@ -40,6 +40,10 @@ "logger": "organizations/123456789098/logs/cloudaudit.googleapis.com%2Fdata_access" }, "related": { + "entity": [ + "organizations/123456789098", + "joel.miller@contoso.com" + ], "user": [ "joel.miller@contoso.com" ] @@ -52,4 +56,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml index e705b8f329a..5db73156fd8 100644 --- a/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml +++ b/packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml @@ -48,7 +48,7 @@ processors: pattern: "%{}%2F%{event.provider}" ignore_missing: true # NOTE test data fails the spec - ignore_failure: true + ignore_failure: true - set: field: event.kind @@ -105,6 +105,94 @@ processors: API_VERSION: (v\d+([a-z]+)?(\d+)?) RESOURCE_TYPE: ([a-z]+((\.[a-z0-9]+)+)?) ignore_missing: true + - script: + description: Appends any relevant entity to `related.entity` for all events + lang: painless + on_failure: + - set: + description: Adds error reason to the document + field: error.message + value: "{{{ _ingest.on_failure_message }}}" + source: | + void addValue(Set entities, def value) { + if (value != null && value != "") { + entities.add(value); + } + } + + boolean isKubernetes = false; + if (ctx.json?.resource?.type != null) { + String typ = ctx.json.resource.type; + isKubernetes = (typ == "k8s_cluster" || typ == "gke_cluster" || typ == "kubernetes"); + } + + // Using tree set to ensure a sorting is kept (testing purposes) + TreeSet entities = new TreeSet(); + + addValue(entities, ctx.json?.protoPayload?.request?.parent); + if (!isKubernetes) { + addValue(entities, ctx.json?.protoPayload?.resourceName); + addValue(entities, ctx.json?.protoPayload?.response?.user); + } + + HashMap authInfo = ctx.json?.protoPayload?.authenticationInfo ?: new HashMap(); + if (!isKubernetes) { + addValue(entities, authInfo.principalEmail); + } + addValue(entities, authInfo.principalSubject); + addValue(entities, authInfo.serviceAccountKeyName); + if (authInfo.serviceAccountDelegationInfo instanceof List) { + for (def i: authInfo.serviceAccountDelegationInfo) { + addValue(entities, i.principalSubject); + addValue(entities, i.firstPartyPrincipal?.principalEmail); + addValue(entities, i.thirdPartyPrincipal?.principalEmail); + } + } + + String serviceName = ctx.json?.protoPayload?.serviceName ?: ''; + if (serviceName == "compute.googleapis.com") { + if (ctx.json?.protoPayload?.request?.networkInterfaces instanceof List) { + for (def e: ctx.json.protoPayload.request.networkInterfaces) { + addValue(entities, e.network); + } + } + if (ctx.json?.protoPayload?.request?.serviceAccounts instanceof List) { + for (def e: ctx.json.protoPayload.request.serviceAccounts) { + addValue(entities, e.email); + } + } + if (ctx.json?.protoPayload?.request?.disks instanceof List) { + for (def e: ctx.json.protoPayload.request.disks) { + addValue(entities, e.source); + } + } + } else if (serviceName == "cloudresourcemanager.googleapis.com") { + if (ctx.json?.protoPayload?.request?.policy?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.request.policy.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } + } + } + if (ctx.json?.protoPayload?.response?.bindings instanceof List) { + for (def e: ctx.json.protoPayload.response.bindings) { + addValue(entities, e.role); + for (def m: e.members) { + addValue(entities, m); + } + } + } + } else if (serviceName == "iamcredentials.googleapis.com") { + if (ctx.json?.protoPayload?.metadata?.identityDelegationChain instanceof List) { + for (def e: ctx.json.protoPayload.metadata.identityDelegationChain) { + addValue(entities, e); + } + } + } + + ctx.related = ctx.related ?: [:]; + ctx.related.entity = entities; ## # AuthenticationInfo @@ -370,9 +458,9 @@ processors: ## # if gcp.audit.authorization_info.[0].granted is true then -# set event.category [network, configuration] and event.type to [access, allowed]; +# set event.category [network, configuration] and event.type to [access, allowed]; # Caveat -# 1. protoPayload.resourceName is a single value while authorization_info[].resource +# 1. protoPayload.resourceName is a single value while authorization_info[].resource # is a list. # 2. as per test data authorization_info may not be as per spec. ## diff --git a/packages/gcp/data_stream/audit/fields/base-fields.yml b/packages/gcp/data_stream/audit/fields/base-fields.yml index 4a7da765108..529f2bfaeee 100644 --- a/packages/gcp/data_stream/audit/fields/base-fields.yml +++ b/packages/gcp/data_stream/audit/fields/base-fields.yml @@ -18,3 +18,10 @@ type: constant_keyword description: Event dataset value: gcp.audit +- name: related.entity + description: | + A collection of all entity identifiers associated with the document. + If the document contains multiple entities, identifiers for each will be included. + Example identifiers include (but not limited to) cloud resource IDs, email addresses, + and hostnames. + type: keyword diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 2f542771070..040dea358f3 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -288,6 +288,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | +| related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | An example event for `audit` looks as following: diff --git a/packages/gcp/docs/audit.md b/packages/gcp/docs/audit.md index 720b5b494e7..c2d8be56e80 100644 --- a/packages/gcp/docs/audit.md +++ b/packages/gcp/docs/audit.md @@ -77,6 +77,7 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur | host.os.codename | OS codename, if any. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | +| related.entity | A collection of all entity identifiers associated with the document. If the document contains multiple entities, identifiers for each will be included. Example identifiers include (but not limited to) cloud resource IDs, email addresses, and hostnames. | keyword | An example event for `audit` looks as following: diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 64276f363b5..539c0e69fdb 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.38.0" +version: "2.39.0" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: