feat(observability): Migrate Observability plugin to kork

gradle.properties

@@ -4,4 +4,4 @@ spinnakerGradleVersion=8.32.1
 targetJava17=true
 includeRuntimes=actuator,core,eureka,retrofit,secrets-aws,secrets-gcp,stackdriver,swagger,tomcat,web
 
-org.gradle.jvmargs=-Xmx2g -Xms2g
+org.gradle.jvmargs=-Xmx4g -Xms4g

kork-core/kork-core.gradle

@@ -25,6 +25,18 @@ dependencies {
   implementation "com.netflix.spectator:spectator-ext-jvm"
   implementation "com.netflix.spectator:spectator-reg-micrometer"
 
+  implementation("io.micrometer:micrometer-registry-prometheus") {
+    exclude group: 'io.micrometer', module: 'micrometer-core'
+  }
+  implementation('io.micrometer:micrometer-registry-datadog') {
+    exclude group: 'io.micrometer', module: 'micrometer-core'
+    exclude group: 'org.slf4j', module: 'slf4j-api'
+  }
+  implementation('com.newrelic.telemetry:micrometer-registry-new-relic:0.5.0') {
+    exclude group: 'io.micrometer', module: 'micrometer-core'
+    exclude group: 'org.slf4j', module: 'slf4j-api'
+  }
+
   testImplementation project(":kork-test")
   testImplementation "org.assertj:assertj-core"
   testImplementation "org.junit.jupiter:junit-jupiter-api"

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/datadog/DataDogRegistryConfig.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.datadog;
+
+import com.netflix.spinnaker.kork.observability.model.MetricsDatadogConfig;
+import java.util.Optional;
+
+public class DataDogRegistryConfig implements io.micrometer.datadog.DatadogConfig {
+  private final MetricsDatadogConfig datadogConfig;
+
+  public DataDogRegistryConfig(MetricsDatadogConfig datadogConfig) {
+    this.datadogConfig = datadogConfig;
+  }
+
+  @Override
+  public String get(String key) {
+    return null; // NOOP, source config from the PluginConfig that is injected
+  }
+
+  @Override
+  public String apiKey() {
+    return Optional.ofNullable(datadogConfig.getApiKey())
+        .orElseThrow(
+            () -> new RuntimeException("The datadog API key is a required plugin config property"));
+  }
+
+  @Override
+  public String applicationKey() {
+    return datadogConfig.getApplicationKey();
+  }
+
+  @Override
+  public String uri() {
+    return datadogConfig.getUri();
+  }
+
+  @Override
+  public boolean descriptions() {
+    return false;
+  }
+
+  @Override
+  public String hostTag() {
+    return "hostTag";
+  }
+
+  @Override
+  public int batchSize() {
+    return datadogConfig.getBatchSize();
+  }
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/datadog/DataDogRegistrySupplier.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.datadog;
+
+import com.netflix.spinnaker.kork.observability.model.MetricsDatadogConfig;
+import com.netflix.spinnaker.kork.observability.model.ObservabilityConfigurationProperites;
+import com.netflix.spinnaker.kork.observability.registry.RegistryConfigWrapper;
+import io.micrometer.core.ipc.http.HttpUrlConnectionSender;
+import io.micrometer.datadog.DatadogMeterRegistry;
+import java.time.Duration;
+import java.util.function.Supplier;
+import org.jetbrains.annotations.NotNull;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class DataDogRegistrySupplier implements Supplier<RegistryConfigWrapper> {
+
+  private final MetricsDatadogConfig datadogConfig;
+  protected HttpUrlConnectionSender sender;
+
+  public DataDogRegistrySupplier(@NotNull ObservabilityConfigurationProperites pluginConfig) {
+    datadogConfig = pluginConfig.getMetrics().getDatadog();
+    this.sender =
+        new HttpUrlConnectionSender(
+            Duration.ofSeconds(datadogConfig.getConnectDurationSeconds()),
+            Duration.ofSeconds(datadogConfig.getReadDurationSeconds()));
+  }
+
+  @Override
+  public RegistryConfigWrapper get() {
+    if (!datadogConfig.isEnabled()) {
+      return null;
+    }
+    var config = new DataDogRegistryConfig(datadogConfig);
+    var registry = DatadogMeterRegistry.builder(config).httpClient(sender).build();
+
+    return RegistryConfigWrapper.builder()
+        .meterRegistry(registry)
+        .meterRegistryConfig(datadogConfig.getRegistry())
+        .build();
+  }
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/filters/ArmoryRecommendedFilters.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.filters;
+
+import static com.netflix.spinnaker.kork.observability.filters.Filters.DENY_CONTROLLER_INVOCATIONS_METRICS;
+
+import io.micrometer.core.instrument.config.MeterFilter;
+import java.util.List;
+
+public class ArmoryRecommendedFilters {
+
+  /*
+   * Curated list of filters/transformations that Armory uses internally to reduce DPM / metrics TS cardinality.
+   * This list is not guaranteed to have backwards compatibility with Sym versioning of the plugin.
+   * What I mean by that is that I will add and remove things to this list that you might rely on without major versioning the plugin.
+   *
+   * TODO implement pattern that allows for users to pick and choose named filters and not rely on the curated list.
+   */
+  public static List<MeterFilter> ARMORY_RECOMMENDED_FILTERS =
+      List.of(DENY_CONTROLLER_INVOCATIONS_METRICS);
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/filters/Filters.java

@@ -0,0 +1,26 @@
+package com.netflix.spinnaker.kork.observability.filters;
+
+import io.micrometer.core.instrument.config.MeterFilter;
+
+public class Filters {
+
+  /*
+   * Removes the spinnaker created 'controller.invocations' metric to prefer the micrometer created 'http.server.requests' metric
+   * They both contain http metrics, however http.server.requests is non-java / sb specific and allows for dashboards that can interoperate x-framework
+   *
+   * https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#production-ready-metrics-spring-mvc
+   *
+   * Extra details are off by default and opt in
+   * The preferred way to add percentiles, percentile histograms, and SLA boundaries is to apply the general
+   * purpose property-based meter filter mechanism to this timer:
+   *
+   * management.metrics.distribution:
+   *  percentiles[http.server.requests]: 0.95, 0.99
+   *  percentiles-histogram[http.server.requests]: true
+   *  sla[http.server.requests]: 10ms, 100ms
+   *
+   * See: https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#per-meter-properties
+   */
+  public static final MeterFilter DENY_CONTROLLER_INVOCATIONS_METRICS =
+      MeterFilter.denyNameStartsWith("controller.invocations");
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/model/ArmoryEnvironmentMetadata.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class ArmoryEnvironmentMetadata {
+  private final String applicationName;
+  private final String ossAppVersion;
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/model/MeterRegistryConfig.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.model;
+
+import java.util.List;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MeterRegistryConfig {
+  private boolean armoryRecommendedFiltersEnabled = false;
+  private boolean defaultTagsDisabled = false;
+
+  private List<String> excludedMetricsPrefix;
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/model/MetricsConfig.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.model;
+
+import java.util.Map;
+import lombok.Data;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+
+@Data
+public class MetricsConfig {
+  private Map<String, String> additionalTags = Map.of();
+  private MetricsPrometheusConfig prometheus = new MetricsPrometheusConfig();
+  private MetricsNewRelicConfig newrelic = new MetricsNewRelicConfig();
+  private MetricsDatadogConfig datadog = new MetricsDatadogConfig();
+  private boolean armoryRecommendedFiltersEnabled = false;
+
+  @Bean
+  @ConditionalOnProperty(value = "observability.config.metrics.prometheus", matchIfMissing = true)
+  public MetricsPrometheusConfig prometheus() {
+    return this.prometheus;
+  }
+
+  @Bean
+  @ConditionalOnProperty(value = "observability.config.metrics.newrelic", matchIfMissing = true)
+  public MetricsNewRelicConfig newrelic() {
+    return this.newrelic;
+  }
+
+  @ConditionalOnProperty(value = "observability.config.metrics.datadog", matchIfMissing = true)
+  @Bean
+  MetricsDatadogConfig datadog() {
+    return this.datadog;
+  }
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/model/MetricsDatadogConfig.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class MetricsDatadogConfig extends MetricsIntegrationConfig {
+  // Datadog Specific Settings
+  @Builder.Default private boolean enabled = false;
+  @Builder.Default private String apiKey = null;
+  @Builder.Default private String applicationKey = null;
+  // @Builder.Default private String hostTag = null;
+  @Builder.Default private String uri = "https://api.datadoghq.com";
+  // @Builder.Default private String descriptions = null;
+
+  // Push Registry Settings
+  @Builder.Default private int stepInSeconds = 30;
+  @Builder.Default private int batchSize = 10000;
+
+  @Builder.Default private int connectDurationSeconds = 5;
+  @Builder.Default private int readDurationSeconds = 5;
+
+  @Builder.Default private String proxyHost = null;
+  @Builder.Default private Integer proxyPort = null;
+}

kork-core/src/main/java/com/netflix/spinnaker/kork/observability/model/MetricsIntegrationConfig.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2025 Netflix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.kork.observability.model;
+
+import lombok.Data;
+
+@Data
+public class MetricsIntegrationConfig {
+  private MeterRegistryConfig registry = new MeterRegistryConfig();
+}