Merge branch 'master' into pbeckwith/add-ebpf

charts/nr-ebpf-agent/Chart.yaml

@@ -13,7 +13,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.10
+version: 0.1.11
 dependencies:
   - name: common-library
     version: 1.3.0

charts/nr-ebpf-agent/README.md

@@ -76,15 +76,19 @@ Options that can be defined globally include `affinity`, `nodeSelector`, `tolera
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Sets all pods' affinities. Can be configured also with `global.affinity` |
 | cluster | string | `""` | Name of the Kubernetes cluster to be monitored. Mandatory. Can be configured with `global.cluster` |
+| dropDataIpServiceNames | bool | `true` | Drop data when service names map to an IP address. |
+| dropDataKubeSystem | bool | `true` | Drop data from the kube-system namespace. |
+| dropDataNewRelic | bool | `true` | Drop data from the newrelic namespace. |
+| dropDataServiceNameRegex | string | `"gmp-.*"` | Define a regex to mach service names to drop. Example "kube-dns|otel-collector|\\bblah\\b" see Golang Docs for Regex syntax https://github.com/google/re2/wiki/Syntax |
 | ebpfAgent.image.pullPolicy | string | `"IfNotPresent"` | The pull policy is defaulted to IfNotPresent, which skips pulling an image if it already exists. If pullPolicy is defined without a specific value, it is also set to Always. |
 | ebpfAgent.image.repository | string | `"us-west1-docker.pkg.dev/pl-dev-infra/nr-ebpf-agent-lp/ebpf-agent"` | eBPF agent image to be deployed. |
-| ebpfAgent.image.tag | string | `"0.0.4"` | The tag of the eBPF agent image to be deployed. |
+| ebpfAgent.image.tag | string | `"0.0.5"` | The tag of the eBPF agent image to be deployed. |
 | ebpfAgent.resources.limits.memory | string | `"2Gi"` | Max memory allocated to the container. |
 | ebpfAgent.resources.requests.cpu | string | `"100m"` | Min CPU allocated to the container. |
 | ebpfAgent.resources.requests.memory | string | `"250Mi"` | Min memory allocated to the container. |
 | ebpfClient.image.pullPolicy | string | `"IfNotPresent"` | The pull policy is defaulted to IfNotPresent, which skips pulling an image if it already exists. If pullPolicy is defined without a specific value, it is set to Always. |
 | ebpfClient.image.repository | string | `"us-west1-docker.pkg.dev/pl-dev-infra/nr-ebpf-agent-lp/ebpf-client"` | eBPF client image to be deployed. |
-| ebpfClient.image.tag | string | `"0.0.6"` | The tag of the eBPF client image to be deployed. |
+| ebpfClient.image.tag | string | `"0.0.7"` | The tag of the eBPF client image to be deployed. |
 | ebpfClient.resources.limits.memory | string | `"100Mi"` | Max memory allocated to the container. |
 | ebpfClient.resources.requests.cpu | string | `"50m"` | Min CPU allocated to the container. |
 | ebpfClient.resources.requests.memory | string | `"50Mi"` | Min memory allocated to the container. |
@@ -101,9 +105,9 @@ Options that can be defined globally include `affinity`, `nodeSelector`, `tolera
 | otelCollector.resources.requests.cpu | string | `"100m"` | Min CPU allocated to the container. |
 | otelCollector.resources.requests.memory | string | `"200Mi"` | Min memory allocated to the container. |
 | podLabels | object | `{}` | Additional labels for chart pods |
-| protocols | object | `{"amqp":true,"cass":true,"dns":true,"http":true,"kafka":true,"mongodb":true,"mysql":true,"pgsql":true,"redis":true}` | The protocols (and data export scripts) to enable for tracing in the socket_tracer. |
+| protocols | object | `{"amqp":{"enabled":true,"samplingLatency":""},"cass":{"enabled":true,"samplingLatency":""},"dns":{"enabled":true,"samplingLatency":""},"http":{"enabled":true,"samplingLatency":""},"kafka":{"enabled":true,"samplingLatency":""},"mongodb":{"enabled":true,"samplingLatency":""},"mysql":{"enabled":true,"samplingLatency":""},"pgsql":{"enabled":true,"samplingLatency":""},"redis":{"enabled":true,"samplingLatency":""}}` | The protocols to enable for tracing in the socket_tracer. samplingLatency represents the sampling latency threshold for the spans to export. Options: p1, p10, p50, p90, p99. |
 | proxy | string | `""` | Configures the agent to send all data through the proxy specified via the otel collector. |
-| pushPeriod | string | `"15"` | The periodicity in seconds at which the eBPF client pushes data to the OTel collector for export to NR. The eBPF agent applies a request path clustering algorithm to reduce cardinality in exported HTTP data. The algorithm only looks for similar request paths within data of the same push period.  To increase the window under consideration for cardinality reduction, increase this value. Accepted range: 15-60. |
+| pushPeriod | string | `"15"` | The periodicity in seconds at which the eBPF agent pushes data to the OTel collector for export to NR. The eBPF agent applies a request path clustering algorithm to reduce cardinality in exported HTTP data. The algorithm only looks for similar request paths within data of the same push period. To increase the window under consideration for cardinality reduction, increase this value. Accepted range: 15-60. |
 | stirlingSources | string | `"socket_tracer,tcp_stats"` | The source connectors (and data export scripts) to enable. Note that socket_tracer tracks http, mysql, redis, mongodb, amqp, cassandra, dns, and postgresql while tcp_stats tracks TCP metrics. |
 | tableStoreDataLimitMB | string | `"250"` | The primary lever to control RAM use of the eBPF agent. Specified in MiB. |
 | tolerations | list | `[]` | Sets all pods' tolerations to node taints. Can be configured also with `global.tolerations` |

charts/nr-ebpf-agent/templates/nr-ebpf-agent-daemonset.yaml

@@ -24,8 +24,8 @@ spec:
           - name: PL_STIRLING_SOURCES
             value: "{{ .Values.stirlingSources }}"
           {{- if .Values.protocols }}
-          {{- range $protocol, $enabled := .Values.protocols }}
-          {{- if (eq $enabled false) }}
+          {{- range $protocol, $config := .Values.protocols }}
+          {{- if (eq $config.enabled false) }}
           - name: PX_STIRLING_ENABLE_{{ upper $protocol }}_TRACING
             value: "0"
           {{- end }}
@@ -62,11 +62,15 @@ spec:
           - name: PL_STIRLING_SOURCES
             value: "{{ .Values.stirlingSources }}"
           {{- if .Values.protocols }}
-          {{- range $protocol, $enabled := .Values.protocols }}
-          {{- if (eq $enabled false) }}
+          {{- range $protocol, $config := .Values.protocols }}
+          {{- if (eq $config.enabled false) }}
           - name: PX_STIRLING_ENABLE_{{ upper $protocol }}_TRACING
             value: "0"
           {{- end }}
+          {{- if (eq $config.enabled true) }}
+          - name: SAMPLE_{{ upper $protocol }}_LATENCY
+            value: {{ $config.samplingLatency | regexMatch "p1|p10|p50|p90|p99" | ternary $config.samplingLatency "p1" }}
+          {{- end }}
           {{- end }}
           {{- end }}
           # TODO(kpattaswamy): Once we implement TLS, we should make this configurable again

charts/nr-ebpf-agent/templates/otel-collector-config.yaml

@@ -73,13 +73,61 @@ data:
               from_attribute: remote_addr
               action: upsert
 
+        filter/nill_service_names: # Drop Data that will be mapped to an empty service name.
+          error_mode: ignore # If for some reason this throws an exception we'll log it and not drop data.
+          traces:
+            span:
+              - 'resource.attributes["k8s.service.name"] == "-" or resource.attributes["k8s.service.name"] == ""'
+              - 'resource.attributes["k8s.service.name"] == nil and (resource.attributes["service.name"] == "-" or resource.attributes["service.name"] == "")'
+          metrics:
+            metric:
+              - 'resource.attributes["k8s.service.name"] == "-" or resource.attributes["k8s.service.name"] == ""'
+              - 'resource.attributes["k8s.service.name"] == nil and (resource.attributes["service.name"] == "-" or resource.attributes["service.name"] == "")'
+
+        filter/kube_system: # Drop data from the kube-service namespace
+          error_mode: ignore # If for some reason this throws an exception we'll log it and not drop data.
+          traces:
+            span:
+              - 'resource.attributes["k8s.namespace.name"] == "kube-system"'
+          metrics:
+            metric:
+              - 'resource.attributes["k8s.namespace.name"] == "kube-system"'
+
+        filter/new_relic: # Drop data from the NewRelic namespace
+          error_mode: ignore # If for some reason this throws an exception we'll log it and not drop data.
+          traces:
+            span:
+              - 'resource.attributes["k8s.namespace.name"] == "newrelic"'
+          metrics:
+            metric:
+              - 'resource.attributes["k8s.namespace.name"] == "newrelic"'
+
+        filter/ip_named_services: # Drop data where the service name maps to an IP address
+          error_mode: ignore # If for some reason this throws an exception we'll log it and not drop data.
+          traces:
+            span:
+              - 'resource.attributes["k8s.service.name"] == nil'
+          metrics:
+            metric:
+              - 'resource.attributes["k8s.service.name"] == nil'
+
+        filter/drop_specific_service_names: # Drop data from namespaces that match the provided regex
+          error_mode: ignore # If for some reason this throws an exception we'll log it and not drop data.
+          traces:
+            span:
+              - 'IsMatch(resource.attributes["k8s.service.name"], "${DROP_SERVICE_NAME_REGEX}")'
+          metrics:
+            metric:
+              - 'IsMatch(resource.attributes["k8s.service.name"], "${DROP_SERVICE_NAME_REGEX}")'
+
         resource/setup_for_export:
           # Setup the resource attr. to export to the endpoint.
           attributes:
             # Rename the k8s.service.name key found from the local ip to peer.service
             - key: peer.service
               from_attribute: k8s.service.name
               action: upsert
+            # TODO(kpattaswamy): Figure out how to set this only when trace_role=server to cause only the client->server direction to render instead of both directions.
             # From the perspective of entity platform, we build the source guid from `parent.service.name` and the target from `service.name`. Hence server-side spans will show the client->server direction.
             - key: parent.service.name
               from_attribute: peer.service
@@ -95,10 +143,6 @@ data:
               action: insert
               value: $CLUSTER_NAME
             # Remove the addr/ports.
-            - key: local_addr
-              action: delete
-            - key: local_port
-              action: delete
             - key: remote_addr
               action: delete
             - key: remote_port
@@ -117,6 +161,19 @@ data:
             processors:
               - resource/setup_for_local_k8s_md
               - k8sattributes/local_k8s_md
+              - filter/nill_service_names
+              {{- if .Values.dropDataKubeSystem}}
+              - filter/kube_system
+              {{- end}}
+              {{- if .Values.dropDataNewRelic}}
+              - filter/new_relic
+              {{- end}}
+              {{- if .Values.dropDataIpServiceNames}}
+              - filter/ip_named_services
+              {{- end}}
+              {{- if .Values.dropDataServiceNameRegex }}
+              - filter/drop_specific_service_names
+              {{- end}}
               - resource/setup_for_remote_k8s_md
               - k8sattributes/remote_k8s_md
               - resource/setup_for_export
@@ -128,6 +185,19 @@ data:
             processors:
               - resource/setup_for_local_k8s_md
               - k8sattributes/local_k8s_md
+              - filter/nill_service_names
+              {{- if .Values.dropDataKubeSystem}}
+              - filter/kube_system
+              {{- end}}
+              {{- if .Values.dropDataNewRelic}}
+              - filter/new_relic
+              {{- end}}
+              {{- if .Values.dropDataIpServiceNames}}
+              - filter/ip_named_services
+              {{- end}}
+              {{- if .Values.dropDataServiceNameRegex }}
+              - filter/drop_specific_service_names
+              {{- end}}
               - resource/setup_for_remote_k8s_md
               - k8sattributes/remote_k8s_md
               - resource/setup_for_export

charts/nr-ebpf-agent/templates/otel-collector-daemonset.yaml

@@ -58,6 +58,8 @@ spec:
               configMapKeyRef:
                 name: {{ include "nr-ebpf-agent.fullname" . }}-config
                 key: clusterName
+          - name: DROP_SERVICE_NAME_REGEX
+            value: "{{- .Values.dropDataServiceNameRegex }}"
         ports:
         - containerPort: 4317
         volumeMounts:

charts/nr-ebpf-agent/values.yaml

@@ -6,23 +6,52 @@ licenseKey: ""
 nrStaging: false
 # -- Configures the agent to send all data through the proxy specified via the otel collector.
 proxy: ""
+# -- Drop data when service names map to an IP address.
+dropDataIpServiceNames: true
+# -- Drop data from the kube-system namespace.
+dropDataKubeSystem: true
+# -- Drop data from the newrelic namespace.
+dropDataNewRelic: true
+# -- Define a regex to mach service names to drop. Example "kube-dns|otel-collector|\\bblah\\b" see Golang Docs for Regex syntax https://github.com/google/re2/wiki/Syntax
+dropDataServiceNameRegex: "gmp-.*"
 # -- The primary lever to control RAM use of the eBPF agent. Specified in MiB.
 tableStoreDataLimitMB: "250"
 # -- The source connectors (and data export scripts) to enable.
 # Note that socket_tracer tracks http, mysql, redis, mongodb, amqp, cassandra, dns, and postgresql
 # while tcp_stats tracks TCP metrics.
 stirlingSources: "socket_tracer,tcp_stats"
-# -- The protocols (and data export scripts) to enable for tracing in the socket_tracer.
+# -- The protocols to enable for tracing in the socket_tracer.
+# samplingLatency represents the sampling latency threshold for the spans to export.
+# Options: p1, p10, p50, p90, p99.
 protocols:
-  http: true
-  kafka: true
-  mysql: true
-  redis: true
-  mongodb: true
-  amqp: true
-  cass: true
-  dns: true
-  pgsql: true
+  http:
+    enabled: true
+    samplingLatency: ""
+  kafka:
+    enabled: true
+    samplingLatency: ""
+  mysql:
+    enabled: true
+    samplingLatency: ""
+  redis:
+    enabled: true
+    samplingLatency: ""
+  mongodb:
+    enabled: true
+    samplingLatency: ""
+  amqp:
+    enabled: true
+    samplingLatency: ""
+  cass:
+    enabled: true
+    samplingLatency: ""
+  dns:
+    enabled: true
+    samplingLatency: ""
+  pgsql:
+    enabled: true
+    samplingLatency: ""
+
 # -- The periodicity in seconds at which the eBPF agent pushes data to the OTel collector for export to NR.
 # The eBPF agent applies a request path clustering algorithm to reduce cardinality in exported HTTP data.
 # The algorithm only looks for similar request paths within data of the same push period.
@@ -37,7 +66,7 @@ ebpfAgent:
     # -- The pull policy is defaulted to IfNotPresent, which skips pulling an image if it already exists. If pullPolicy is defined without a specific value, it is also set to Always.
     pullPolicy: IfNotPresent
     # -- The tag of the eBPF agent image to be deployed.
-    tag: 0.0.4
+    tag: 0.0.5
   resources:
     limits:
       # -- Max memory allocated to the container.
@@ -56,7 +85,7 @@ ebpfClient:
     # -- The pull policy is defaulted to IfNotPresent, which skips pulling an image if it already exists. If pullPolicy is defined without a specific value, it is set to Always.
     pullPolicy: IfNotPresent
     # -- The tag of the eBPF client image to be deployed.
-    tag: 0.0.6
+    tag: 0.0.7
   resources:
     limits:
       # -- Max memory allocated to the container.

charts/nri-bundle/Chart.lock

@@ -1,7 +1,7 @@
 dependencies:
 - name: newrelic-infrastructure
   repository: https://newrelic.github.io/nri-kubernetes
-  version: 3.35.1
+  version: 3.36.0
 - name: nri-prometheus
   repository: https://newrelic.github.io/nri-prometheus
   version: 2.1.19
@@ -13,7 +13,7 @@ dependencies:
   version: 4.22.1
 - name: newrelic-k8s-metrics-adapter
   repository: https://newrelic.github.io/newrelic-k8s-metrics-adapter
-  version: 1.12.1
+  version: 1.13.0
 - name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
   version: 5.26.0
@@ -22,7 +22,7 @@ dependencies:
   version: 3.11.1
 - name: newrelic-logging
   repository: https://newrelic.github.io/helm-charts
-  version: 1.23.3
+  version: 1.23.5
 - name: newrelic-pixie
   repository: https://newrelic.github.io/helm-charts
   version: 2.1.6
@@ -31,12 +31,12 @@ dependencies:
   version: 0.1.10
 - name: k8s-agents-operator
   repository: https://newrelic.github.io/k8s-agents-operator
-  version: 0.17.0
+  version: 0.18.0
 - name: pixie-operator-chart
   repository: https://pixie-operator-charts.storage.googleapis.com
   version: 0.1.6
 - name: newrelic-infra-operator
   repository: https://newrelic.github.io/newrelic-infra-operator
   version: 2.13.0
-digest: sha256:27f34a3fbdf61a25b209a9005c428aeda57f8f945ca0349d3b051f88e221d532
-generated: "2024-11-07T13:01:33.265062-08:00"
+digest: sha256:26318e3603fa6f822c2139751f99017adb7bd47de5a02f19c308426d707ab3b0
+generated: "2024-11-11T14:04:24.402496325Z"

charts/nri-bundle/Chart.yaml

@@ -24,7 +24,7 @@ dependencies:
   - name: newrelic-infrastructure
     repository: https://newrelic.github.io/nri-kubernetes
     condition: infrastructure.enabled,newrelic-infrastructure.enabled
-    version: 3.35.1
+    version: 3.36.0
 
   - name: nri-prometheus
     repository: https://newrelic.github.io/nri-prometheus
@@ -44,7 +44,7 @@ dependencies:
   - name: newrelic-k8s-metrics-adapter
     repository: https://newrelic.github.io/newrelic-k8s-metrics-adapter
     condition: metrics-adapter.enabled,newrelic-k8s-metrics-adapter.enabled
-    version: 1.12.1
+    version: 1.13.0
 
   - name: kube-state-metrics
     version: 5.26.0
@@ -59,7 +59,7 @@ dependencies:
   - name: newrelic-logging
     repository: https://newrelic.github.io/helm-charts
     condition: logging.enabled,newrelic-logging.enabled
-    version: 1.23.3
+    version: 1.23.5
 
   - name: newrelic-pixie
     repository: https://newrelic.github.io/helm-charts
@@ -74,7 +74,7 @@ dependencies:
   - name: k8s-agents-operator
     repository: https://newrelic.github.io/k8s-agents-operator
     condition: k8s-agents-operator.enabled
-    version: 0.17.0
+    version: 0.18.0
 
   # Keep the version of pixie-operator-chart in sync with the CRD versions for
   # olm_crd.yaml and px.dev_viziers.yaml in