Merge branch 'main' into remove-assumed-esql-features

thecoop · Jan 15, 2025 · 9bbc715 · 9bbc715
2 parents 941d50b + a2d84b1
commit 9bbc715
Show file tree

Hide file tree

Showing 580 changed files with 6,514 additions and 5,430 deletions.
diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
+if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false"  || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then
   exit 0
 fi
 

diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java
@@ -27,6 +27,7 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
+import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
@@ -71,12 +72,11 @@ public class EvalBenchmark {
         BigArrays.NON_RECYCLING_INSTANCE
     );
 
+    private static final FoldContext FOLD_CONTEXT = FoldContext.small();
+
     private static final int BLOCK_LENGTH = 8 * 1024;
 
-    static final DriverContext driverContext = new DriverContext(
-        BigArrays.NON_RECYCLING_INSTANCE,
-        BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE)
-    );
+    static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory);
 
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
@@ -114,18 +114,20 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
         return switch (operation) {
             case "abs" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "add" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
             case "add_double" -> {
                 FieldAttribute doubleField = doubleField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)),
                     layout(doubleField)
                 ).get(driverContext);
@@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                     rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
                 }
-                yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2))
+                    .get(driverContext);
             }
             case "date_trunc" -> {
                 FieldAttribute timestamp = new FieldAttribute(
@@ -149,35 +152,37 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                     new EsField("timestamp", DataType.DATETIME, Map.of(), true)
                 );
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
                     layout(timestamp)
                 ).get(driverContext);
             }
             case "equal_to_const" -> {
                 FieldAttribute longField = longField();
                 yield EvalMapper.toEvaluator(
+                    FOLD_CONTEXT,
                     new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)),
                     layout(longField)
                 ).get(driverContext);
             }
             case "long_equal_to_long" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = longField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "long_equal_to_int" -> {
                 FieldAttribute lhs = longField();
                 FieldAttribute rhs = intField();
-                yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext);
             }
             case "mv_min", "mv_min_ascending" -> {
                 FieldAttribute longField = longField();
-                yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext);
             }
             case "rlike" -> {
                 FieldAttribute keywordField = keywordField();
                 RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
-                yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext);
+                yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
             }
             default -> throw new UnsupportedOperationException();
         };

diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java
@@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) {
     static void extractSecureSettings(SecureSettings secrets, Map<String, String> propertiesMap) {
         final Set<String> settingNames = secrets.getSettingNames();
         for (String key : List.of("api_key", "secret_token")) {
-            for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-                if (settingNames.contains(prefix + key)) {
-                    if (propertiesMap.containsKey(key)) {
-                        throw new IllegalStateException(
-                            Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key)
-                        );
-                    }
-
-                    try (SecureString token = secrets.getString(prefix + key)) {
-                        propertiesMap.put(key, token.toString());
-                    }
+            String prefix = "telemetry.";
+            if (settingNames.contains(prefix + key)) {
+                try (SecureString token = secrets.getString(prefix + key)) {
+                    propertiesMap.put(key, token.toString());
                 }
             }
-
         }
     }
 
@@ -227,44 +219,12 @@ private static Map<String, String> extractDynamicSettings(Map<String, String> pr
     static Map<String, String> extractApmSettings(Settings settings) throws UserException {
         final Map<String, String> propertiesMap = new HashMap<>();
 
-        // tracing.apm.agent. is deprecated by telemetry.agent.
         final String telemetryAgentPrefix = "telemetry.agent.";
-        final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent.";
 
         final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix);
         telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key))));
 
-        final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix);
-        for (String key : apmAgentSettings.keySet()) {
-            if (propertiesMap.containsKey(key)) {
-                throw new IllegalStateException(
-                    Strings.format(
-                        "Duplicate telemetry setting: [%s%s] and [%s%s]",
-                        telemetryAgentPrefix,
-                        key,
-                        deprecatedTelemetryAgentPrefix,
-                        key
-                    )
-                );
-            }
-            propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key)));
-        }
-
         StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings);
-        if (globalLabels.length() == 0) {
-            globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-        } else {
-            StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings);
-            if (tracingGlobalLabels.length() != 0) {
-                throw new IllegalArgumentException(
-                    "Cannot have global labels with tracing.agent prefix ["
-                        + globalLabels
-                        + "] and telemetry.apm.agent prefix ["
-                        + tracingGlobalLabels
-                        + "]"
-                );
-            }
-        }
         if (globalLabels.length() > 0) {
             propertiesMap.put("global_labels", globalLabels.toString());
         }
@@ -274,7 +234,7 @@ static Map<String, String> extractApmSettings(Settings settings) throws UserExce
             if (propertiesMap.containsKey(key)) {
                 throw new UserException(
                     ExitCodes.CONFIG,
-                    "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch"
+                    "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch"
                 );
             }
         }

diff --git a/...ution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/...ution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java
@@ -25,18 +25,15 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.function.Function;
 
 import static org.elasticsearch.test.MapMatcher.matchesMap;
 import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.is;
 import static org.hamcrest.Matchers.not;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
@@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException {
     }
 
     public void testExtractSecureSettings() {
-        MockSecureSettings duplicateSecureSettings = new MockSecureSettings();
+        MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("telemetry.secret_token", "token");
+        secureSettings.setString("telemetry.api_key", "key");
 
-        for (String prefix : List.of("telemetry.", "tracing.apm.")) {
-            MockSecureSettings secureSettings = new MockSecureSettings();
-            secureSettings.setString(prefix + "secret_token", "token");
-            secureSettings.setString(prefix + "api_key", "key");
-
-            duplicateSecureSettings.setString(prefix + "api_key", "secret");
-
-            Map<String, String> propertiesMap = new HashMap<>();
-            APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
-
-            assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
-        }
-
-        Exception exception = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>())
-        );
-        assertThat(exception.getMessage(), containsString("Duplicate telemetry setting"));
-        assertThat(exception.getMessage(), containsString("telemetry.api_key"));
-        assertThat(exception.getMessage(), containsString("tracing.apm.api_key"));
+        Map<String, String> propertiesMap = new HashMap<>();
+        APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap);
 
+        assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key")));
     }
 
     public void testExtractSettings() throws UserException {
-        Function<String, Settings.Builder> buildSettings = (prefix) -> Settings.builder()
-            .put(prefix + "server_url", "https://myurl:443")
-            .put(prefix + "service_node_name", "instance-0000000001");
-
-        for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) {
-            var name = "APM Tracing";
-            var deploy = "123";
-            var org = "456";
-            var extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-
-            assertThat(
-                extracted,
-                allOf(
-                    hasEntry("server_url", "https://myurl:443"),
-                    hasEntry("service_node_name", "instance-0000000001"),
-                    hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
-                    not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
-                )
-            );
-
-            List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(3));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
-
-            // test replacing with underscores and skipping empty
-            name = "APM=Tracing";
-            deploy = "";
-            org = ",456";
-            extracted = APMJvmOptions.extractApmSettings(
-                buildSettings.apply(prefix)
-                    .put(prefix + "global_labels.deployment_name", name)
-                    .put(prefix + "global_labels.deployment_id", deploy)
-                    .put(prefix + "global_labels.organization_id", org)
-                    .build()
-            );
-            labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
-            assertThat(labels, hasSize(2));
-            assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
-        }
-
-        IllegalStateException err = expectThrows(
-            IllegalStateException.class,
-            () -> APMJvmOptions.extractApmSettings(
-                Settings.builder()
-                    .put("tracing.apm.agent.server_url", "https://myurl:443")
-                    .put("telemetry.agent.server_url", "https://myurl-2:443")
-                    .build()
-            )
-        );
-        assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]"));
-    }
-
-    public void testNoMixedLabels() {
-        String telemetryAgent = "telemetry.agent.";
-        String tracingAgent = "tracing.apm.agent.";
-        Settings settings = Settings.builder()
-            .put("tracing.apm.enabled", true)
-            .put(telemetryAgent + "server_url", "https://myurl:443")
-            .put(telemetryAgent + "service_node_name", "instance-0000000001")
-            .put(tracingAgent + "global_labels.deployment_id", "123")
-            .put(telemetryAgent + "global_labels.organization_id", "456")
+        Settings defaults = Settings.builder()
+            .put("telemetry.agent.server_url", "https://myurl:443")
+            .put("telemetry.agent.service_node_name", "instance-0000000001")
             .build();
 
-        IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings));
+        var name = "APM Tracing";
+        var deploy = "123";
+        var org = "456";
+        var extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+
         assertThat(
-            err.getMessage(),
-            is(
-                "Cannot have global labels with tracing.agent prefix [organization_id=456] and"
-                    + " telemetry.apm.agent prefix [deployment_id=123]"
+            extracted,
+            allOf(
+                hasEntry("server_url", "https://myurl:443"),
+                hasEntry("service_node_name", "instance-0000000001"),
+                hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one
+                not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys
             )
         );
+
+        List<String> labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(3));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy));
+
+        // test replacing with underscores and skipping empty
+        name = "APM=Tracing";
+        deploy = "";
+        org = ",456";
+        extracted = APMJvmOptions.extractApmSettings(
+            Settings.builder()
+                .put(defaults)
+                .put("telemetry.agent.global_labels.deployment_name", name)
+                .put("telemetry.agent.global_labels.deployment_id", deploy)
+                .put("telemetry.agent.global_labels.organization_id", org)
+                .build()
+        );
+        labels = Arrays.stream(extracted.get("global_labels").split(",")).toList();
+        assertThat(labels, hasSize(2));
+        assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456"));
     }
 
     private Path makeFakeAgentJar() throws IOException {

diff --git a/docs/changelog/118602.yaml b/docs/changelog/118602.yaml
@@ -0,0 +1,5 @@
+pr: 118602
+summary: Limit memory usage of `fold`
+area: ES|QL
+type: bug
+issues: []
diff --git a/docs/changelog/119227.yaml b/docs/changelog/119227.yaml
@@ -0,0 +1,13 @@
+pr: 119227
+summary: Remove unfreeze REST endpoint
+area: Indices APIs
+type: breaking
+issues: []
+breaking:
+  title: Remove unfreeze REST endpoint
+  area: REST API
+  details: >-
+    The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding
+    `/{index}/_freeze` endpoint was removed in 8.0.
+  impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0
+  notable: false
diff --git a/docs/changelog/119772.yaml b/docs/changelog/119772.yaml
@@ -0,0 +1,6 @@
+pr: 119772
+summary: ESQL Support IN operator for Date nanos
+area: ES|QL
+type: enhancement
+issues:
+ - 118578
diff --git a/docs/changelog/119831.yaml b/docs/changelog/119831.yaml
@@ -0,0 +1,5 @@
+pr: 119831
+summary: Run `TransportClusterGetSettingsAction` on local node
+area: Infra/Settings
+type: enhancement
+issues: []