diff --git a/.buildkite/scripts/get-latest-test-mutes.sh b/.buildkite/scripts/get-latest-test-mutes.sh index 5721e29f1b773..1dafcebec24b1 100755 --- a/.buildkite/scripts/get-latest-test-mutes.sh +++ b/.buildkite/scripts/get-latest-test-mutes.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [[ ! "${BUILDKITE_PULL_REQUEST:-}" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then +if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" || "${BUILDKITE_AGENT_META_DATA_PROVIDER:-}" == "k8s" ]]; then exit 0 fi diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java index 9aab4a3e3210f..d3259b9604717 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java @@ -27,6 +27,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -71,12 +72,11 @@ public class EvalBenchmark { BigArrays.NON_RECYCLING_INSTANCE ); + private static final FoldContext FOLD_CONTEXT = FoldContext.small(); + private static final int BLOCK_LENGTH = 8 * 1024; - static final DriverContext driverContext = new DriverContext( - BigArrays.NON_RECYCLING_INSTANCE, - BlockFactory.getInstance(new NoopCircuitBreaker("noop"), BigArrays.NON_RECYCLING_INSTANCE) - ); + static final DriverContext driverContext = new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, blockFactory); static { // Smoke test all the expected values and force loading subclasses more like prod @@ -114,11 +114,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { return switch (operation) { case "abs" -> { FieldAttribute longField = longField(); - yield EvalMapper.toEvaluator(new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Abs(Source.EMPTY, longField), layout(longField)).get(driverContext); } case "add" -> { FieldAttribute longField = longField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Add(Source.EMPTY, longField, new Literal(Source.EMPTY, 1L, DataType.LONG)), layout(longField) ).get(driverContext); @@ -126,6 +127,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "add_double" -> { FieldAttribute doubleField = doubleField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Add(Source.EMPTY, doubleField, new Literal(Source.EMPTY, 1D, DataType.DOUBLE)), layout(doubleField) ).get(driverContext); @@ -140,7 +142,8 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG)); } - yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)) + .get(driverContext); } case "date_trunc" -> { FieldAttribute timestamp = new FieldAttribute( @@ -149,6 +152,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { new EsField("timestamp", DataType.DATETIME, Map.of(), true) ); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp), layout(timestamp) ).get(driverContext); @@ -156,6 +160,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "equal_to_const" -> { FieldAttribute longField = longField(); yield EvalMapper.toEvaluator( + FOLD_CONTEXT, new Equals(Source.EMPTY, longField, new Literal(Source.EMPTY, 100_000L, DataType.LONG)), layout(longField) ).get(driverContext); @@ -163,21 +168,21 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) { case "long_equal_to_long" -> { FieldAttribute lhs = longField(); FieldAttribute rhs = longField(); - yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); } case "long_equal_to_int" -> { FieldAttribute lhs = longField(); FieldAttribute rhs = intField(); - yield EvalMapper.toEvaluator(new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new Equals(Source.EMPTY, lhs, rhs), layout(lhs, rhs)).get(driverContext); } case "mv_min", "mv_min_ascending" -> { FieldAttribute longField = longField(); - yield EvalMapper.toEvaluator(new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, new MvMin(Source.EMPTY, longField), layout(longField)).get(driverContext); } case "rlike" -> { FieldAttribute keywordField = keywordField(); RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar")); - yield EvalMapper.toEvaluator(rlike, layout(keywordField)).get(driverContext); + yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext); } default -> throw new UnsupportedOperationException(); }; diff --git a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java index c3b9768946767..1e57d9fab7cfd 100644 --- a/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java +++ b/distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/APMJvmOptions.java @@ -187,20 +187,12 @@ static String agentCommandLineOption(Path agentJar, Path tmpPropertiesFile) { static void extractSecureSettings(SecureSettings secrets, Map propertiesMap) { final Set settingNames = secrets.getSettingNames(); for (String key : List.of("api_key", "secret_token")) { - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - if (settingNames.contains(prefix + key)) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format("Duplicate telemetry setting: [telemetry.%s] and [tracing.apm.%s]", key, key) - ); - } - - try (SecureString token = secrets.getString(prefix + key)) { - propertiesMap.put(key, token.toString()); - } + String prefix = "telemetry."; + if (settingNames.contains(prefix + key)) { + try (SecureString token = secrets.getString(prefix + key)) { + propertiesMap.put(key, token.toString()); } } - } } @@ -227,44 +219,12 @@ private static Map extractDynamicSettings(Map pr static Map extractApmSettings(Settings settings) throws UserException { final Map propertiesMap = new HashMap<>(); - // tracing.apm.agent. is deprecated by telemetry.agent. final String telemetryAgentPrefix = "telemetry.agent."; - final String deprecatedTelemetryAgentPrefix = "tracing.apm.agent."; final Settings telemetryAgentSettings = settings.getByPrefix(telemetryAgentPrefix); telemetryAgentSettings.keySet().forEach(key -> propertiesMap.put(key, String.valueOf(telemetryAgentSettings.get(key)))); - final Settings apmAgentSettings = settings.getByPrefix(deprecatedTelemetryAgentPrefix); - for (String key : apmAgentSettings.keySet()) { - if (propertiesMap.containsKey(key)) { - throw new IllegalStateException( - Strings.format( - "Duplicate telemetry setting: [%s%s] and [%s%s]", - telemetryAgentPrefix, - key, - deprecatedTelemetryAgentPrefix, - key - ) - ); - } - propertiesMap.put(key, String.valueOf(apmAgentSettings.get(key))); - } - StringJoiner globalLabels = extractGlobalLabels(telemetryAgentPrefix, propertiesMap, settings); - if (globalLabels.length() == 0) { - globalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - } else { - StringJoiner tracingGlobalLabels = extractGlobalLabels(deprecatedTelemetryAgentPrefix, propertiesMap, settings); - if (tracingGlobalLabels.length() != 0) { - throw new IllegalArgumentException( - "Cannot have global labels with tracing.agent prefix [" - + globalLabels - + "] and telemetry.apm.agent prefix [" - + tracingGlobalLabels - + "]" - ); - } - } if (globalLabels.length() > 0) { propertiesMap.put("global_labels", globalLabels.toString()); } @@ -274,7 +234,7 @@ static Map extractApmSettings(Settings settings) throws UserExce if (propertiesMap.containsKey(key)) { throw new UserException( ExitCodes.CONFIG, - "Do not set a value for [tracing.apm.agent." + key + "], as this is configured automatically by Elasticsearch" + "Do not set a value for [telemetry.agent." + key + "], as this is configured automatically by Elasticsearch" ); } } diff --git a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java index a7ba8eb11fbcc..0e067afc1aa73 100644 --- a/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java +++ b/distribution/tools/server-cli/src/test/java/org/elasticsearch/server/cli/APMJvmOptionsTests.java @@ -25,18 +25,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; import static org.elasticsearch.test.MapMatcher.matchesMap; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsInAnyOrder; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -82,109 +79,63 @@ public void testFileDeleteWorks() throws IOException { } public void testExtractSecureSettings() { - MockSecureSettings duplicateSecureSettings = new MockSecureSettings(); + MockSecureSettings secureSettings = new MockSecureSettings(); + secureSettings.setString("telemetry.secret_token", "token"); + secureSettings.setString("telemetry.api_key", "key"); - for (String prefix : List.of("telemetry.", "tracing.apm.")) { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(prefix + "secret_token", "token"); - secureSettings.setString(prefix + "api_key", "key"); - - duplicateSecureSettings.setString(prefix + "api_key", "secret"); - - Map propertiesMap = new HashMap<>(); - APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); - - assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); - } - - Exception exception = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractSecureSettings(duplicateSecureSettings, new HashMap<>()) - ); - assertThat(exception.getMessage(), containsString("Duplicate telemetry setting")); - assertThat(exception.getMessage(), containsString("telemetry.api_key")); - assertThat(exception.getMessage(), containsString("tracing.apm.api_key")); + Map propertiesMap = new HashMap<>(); + APMJvmOptions.extractSecureSettings(secureSettings, propertiesMap); + assertThat(propertiesMap, matchesMap(Map.of("secret_token", "token", "api_key", "key"))); } public void testExtractSettings() throws UserException { - Function buildSettings = (prefix) -> Settings.builder() - .put(prefix + "server_url", "https://myurl:443") - .put(prefix + "service_node_name", "instance-0000000001"); - - for (String prefix : List.of("tracing.apm.agent.", "telemetry.agent.")) { - var name = "APM Tracing"; - var deploy = "123"; - var org = "456"; - var extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - - assertThat( - extracted, - allOf( - hasEntry("server_url", "https://myurl:443"), - hasEntry("service_node_name", "instance-0000000001"), - hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one - not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys - ) - ); - - List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(3)); - assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); - - // test replacing with underscores and skipping empty - name = "APM=Tracing"; - deploy = ""; - org = ",456"; - extracted = APMJvmOptions.extractApmSettings( - buildSettings.apply(prefix) - .put(prefix + "global_labels.deployment_name", name) - .put(prefix + "global_labels.deployment_id", deploy) - .put(prefix + "global_labels.organization_id", org) - .build() - ); - labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); - assertThat(labels, hasSize(2)); - assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); - } - - IllegalStateException err = expectThrows( - IllegalStateException.class, - () -> APMJvmOptions.extractApmSettings( - Settings.builder() - .put("tracing.apm.agent.server_url", "https://myurl:443") - .put("telemetry.agent.server_url", "https://myurl-2:443") - .build() - ) - ); - assertThat(err.getMessage(), is("Duplicate telemetry setting: [telemetry.agent.server_url] and [tracing.apm.agent.server_url]")); - } - - public void testNoMixedLabels() { - String telemetryAgent = "telemetry.agent."; - String tracingAgent = "tracing.apm.agent."; - Settings settings = Settings.builder() - .put("tracing.apm.enabled", true) - .put(telemetryAgent + "server_url", "https://myurl:443") - .put(telemetryAgent + "service_node_name", "instance-0000000001") - .put(tracingAgent + "global_labels.deployment_id", "123") - .put(telemetryAgent + "global_labels.organization_id", "456") + Settings defaults = Settings.builder() + .put("telemetry.agent.server_url", "https://myurl:443") + .put("telemetry.agent.service_node_name", "instance-0000000001") .build(); - IllegalArgumentException err = assertThrows(IllegalArgumentException.class, () -> APMJvmOptions.extractApmSettings(settings)); + var name = "APM Tracing"; + var deploy = "123"; + var org = "456"; + var extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + assertThat( - err.getMessage(), - is( - "Cannot have global labels with tracing.agent prefix [organization_id=456] and" - + " telemetry.apm.agent prefix [deployment_id=123]" + extracted, + allOf( + hasEntry("server_url", "https://myurl:443"), + hasEntry("service_node_name", "instance-0000000001"), + hasEntry(equalTo("global_labels"), not(endsWith(","))), // test that we have collapsed all global labels into one + not(hasKey("global_labels.organization_id")) // tests that we strip out the top level label keys ) ); + + List labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(3)); + assertThat(labels, containsInAnyOrder("deployment_name=APM Tracing", "organization_id=" + org, "deployment_id=" + deploy)); + + // test replacing with underscores and skipping empty + name = "APM=Tracing"; + deploy = ""; + org = ",456"; + extracted = APMJvmOptions.extractApmSettings( + Settings.builder() + .put(defaults) + .put("telemetry.agent.global_labels.deployment_name", name) + .put("telemetry.agent.global_labels.deployment_id", deploy) + .put("telemetry.agent.global_labels.organization_id", org) + .build() + ); + labels = Arrays.stream(extracted.get("global_labels").split(",")).toList(); + assertThat(labels, hasSize(2)); + assertThat(labels, containsInAnyOrder("deployment_name=APM_Tracing", "organization_id=_456")); } private Path makeFakeAgentJar() throws IOException { diff --git a/docs/changelog/118602.yaml b/docs/changelog/118602.yaml new file mode 100644 index 0000000000000..a75c5dcf11da3 --- /dev/null +++ b/docs/changelog/118602.yaml @@ -0,0 +1,5 @@ +pr: 118602 +summary: Limit memory usage of `fold` +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/119227.yaml b/docs/changelog/119227.yaml new file mode 100644 index 0000000000000..1e3d4f97a3d27 --- /dev/null +++ b/docs/changelog/119227.yaml @@ -0,0 +1,13 @@ +pr: 119227 +summary: Remove unfreeze REST endpoint +area: Indices APIs +type: breaking +issues: [] +breaking: + title: Remove unfreeze REST endpoint + area: REST API + details: >- + The `/{index}/_unfreeze` REST endpoint is no longer supported. This API was deprecated, and the corresponding + `/{index}/_freeze` endpoint was removed in 8.0. + impact: None, since it is not possible to have a frozen index in a version which is readable by Elasticsearch 9.0 + notable: false diff --git a/docs/changelog/119772.yaml b/docs/changelog/119772.yaml new file mode 100644 index 0000000000000..58d483566b109 --- /dev/null +++ b/docs/changelog/119772.yaml @@ -0,0 +1,6 @@ +pr: 119772 +summary: ESQL Support IN operator for Date nanos +area: ES|QL +type: enhancement +issues: + - 118578 diff --git a/docs/changelog/119831.yaml b/docs/changelog/119831.yaml new file mode 100644 index 0000000000000..61c09d7d54de0 --- /dev/null +++ b/docs/changelog/119831.yaml @@ -0,0 +1,5 @@ +pr: 119831 +summary: Run `TransportClusterGetSettingsAction` on local node +area: Infra/Settings +type: enhancement +issues: [] diff --git a/docs/changelog/119846.yaml b/docs/changelog/119846.yaml new file mode 100644 index 0000000000000..9e7d99fe1be13 --- /dev/null +++ b/docs/changelog/119846.yaml @@ -0,0 +1,12 @@ +pr: 119846 +summary: Drop support for brackets from METADATA syntax +area: ES|QL +type: deprecation +issues: + - 115401 +deprecation: + title: Drop support for brackets from METADATA syntax + area: ES|QL + details: Please describe the details of this change for the release notes. You can + use asciidoc. + impact: Please describe the impact of this change to users diff --git a/docs/changelog/119893.yaml b/docs/changelog/119893.yaml new file mode 100644 index 0000000000000..35a46ce0940d3 --- /dev/null +++ b/docs/changelog/119893.yaml @@ -0,0 +1,5 @@ +pr: 119893 +summary: Add enterprise license check for Inference API actions +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/119922.yaml b/docs/changelog/119922.yaml new file mode 100644 index 0000000000000..2fc9d9529c968 --- /dev/null +++ b/docs/changelog/119922.yaml @@ -0,0 +1,5 @@ +pr: 119922 +summary: "[Inference API] fix spell words: covertToString to convertToString" +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/119926.yaml b/docs/changelog/119926.yaml new file mode 100644 index 0000000000000..3afafd5b2117f --- /dev/null +++ b/docs/changelog/119926.yaml @@ -0,0 +1,11 @@ +pr: 119926 +summary: "Deprecated tracing.apm.* settings got removed." +area: Infra/Metrics +type: breaking +issues: [] +breaking: + title: "Deprecated tracing.apm.* settings got removed." + area: Cluster and node setting + details: Deprecated `tracing.apm.*` settings got removed, use respective `telemetry.*` / `telemetry.tracing.*` settings instead. + impact: 9.x nodes will refuse to start if any such setting (including secret settings) is still present. + notable: false diff --git a/docs/changelog/120014.yaml b/docs/changelog/120014.yaml new file mode 100644 index 0000000000000..bef1f3ba49939 --- /dev/null +++ b/docs/changelog/120014.yaml @@ -0,0 +1,6 @@ +pr: 120014 +summary: Fix potential file leak in ES816BinaryQuantizedVectorsWriter +area: Search +type: bug +issues: + - 119981 diff --git a/docs/changelog/120038.yaml b/docs/changelog/120038.yaml new file mode 100644 index 0000000000000..fe3a2ccccc095 --- /dev/null +++ b/docs/changelog/120038.yaml @@ -0,0 +1,5 @@ +pr: 120038 +summary: Run template simulation actions on local node +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/changelog/120042.yaml b/docs/changelog/120042.yaml new file mode 100644 index 0000000000000..0093068ae9894 --- /dev/null +++ b/docs/changelog/120042.yaml @@ -0,0 +1,5 @@ +pr: 120042 +summary: Match dot prefix of migrated DS backing index with the source index +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/120062.yaml b/docs/changelog/120062.yaml new file mode 100644 index 0000000000000..42e8d97f17444 --- /dev/null +++ b/docs/changelog/120062.yaml @@ -0,0 +1,6 @@ +pr: 120062 +summary: Update Text Similarity Reranker to Properly Handle Aliases +area: Ranking +type: bug +issues: + - 119617 diff --git a/docs/changelog/120084.yaml b/docs/changelog/120084.yaml new file mode 100644 index 0000000000000..aafe490d79f1e --- /dev/null +++ b/docs/changelog/120084.yaml @@ -0,0 +1,5 @@ +pr: 120084 +summary: Improve how reindex data stream index action handles api blocks +area: Data streams +type: enhancement +issues: [] diff --git a/docs/changelog/120087.yaml b/docs/changelog/120087.yaml new file mode 100644 index 0000000000000..8539640809b04 --- /dev/null +++ b/docs/changelog/120087.yaml @@ -0,0 +1,5 @@ +pr: 120087 +summary: Include `clusterApplyListener` in long cluster apply warnings +area: Cluster Coordination +type: enhancement +issues: [] diff --git a/docs/changelog/120133.yaml b/docs/changelog/120133.yaml new file mode 100644 index 0000000000000..4ec88267a1bf8 --- /dev/null +++ b/docs/changelog/120133.yaml @@ -0,0 +1,6 @@ +pr: 120133 +summary: Use approximation to advance matched queries +area: Search +type: bug +issues: + - 120130 diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc index 8ce8064a8161c..adfd38478ab21 100644 --- a/docs/reference/esql/esql-limitations.asciidoc +++ b/docs/reference/esql/esql-limitations.asciidoc @@ -30,11 +30,11 @@ include::processing-commands/limit.asciidoc[tag=limitation] ** You can use `to_datetime` to cast to millisecond dates to use unsupported functions * `double` (`float`, `half_float`, `scaled_float` are represented as `double`) * `ip` -* `keyword` family including `keyword`, `constant_keyword`, and `wildcard` +* `keyword` <> including `keyword`, `constant_keyword`, and `wildcard` * `int` (`short` and `byte` are represented as `int`) * `long` * `null` -* `text` +* `text` <> including `text`, `semantic_text` and `match_only_text` * experimental:[] `unsigned_long` * `version` * Spatial types diff --git a/docs/reference/esql/functions/description/match.asciidoc b/docs/reference/esql/functions/description/match.asciidoc index 931fd5eb2f94a..0724f0f108e3c 100644 --- a/docs/reference/esql/functions/description/match.asciidoc +++ b/docs/reference/esql/functions/description/match.asciidoc @@ -2,4 +2,4 @@ *Description* -Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. +Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. Match can be used on fields from the text family like <> and <>, as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. `MATCH` returns true if the provided query matches the row. diff --git a/docs/reference/esql/functions/description/to_date_nanos.asciidoc b/docs/reference/esql/functions/description/to_date_nanos.asciidoc index 3fac7295f1bed..955c19b43a12f 100644 --- a/docs/reference/esql/functions/description/to_date_nanos.asciidoc +++ b/docs/reference/esql/functions/description/to_date_nanos.asciidoc @@ -4,4 +4,4 @@ Converts an input to a nanosecond-resolution date value (aka date_nanos). -NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. +NOTE: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning.. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/kibana/definition/bucket.json b/docs/reference/esql/functions/kibana/definition/bucket.json index 3d96de05c8407..f9c7f2f27d6f9 100644 --- a/docs/reference/esql/functions/kibana/definition/bucket.json +++ b/docs/reference/esql/functions/kibana/definition/bucket.json @@ -1599,7 +1599,7 @@ "FROM sample_data \n| WHERE @timestamp >= NOW() - 1 day and @timestamp < NOW()\n| STATS COUNT(*) BY bucket = BUCKET(@timestamp, 25, NOW() - 1 day, NOW())", "FROM employees\n| WHERE hire_date >= \"1985-01-01T00:00:00Z\" AND hire_date < \"1986-01-01T00:00:00Z\"\n| STATS AVG(salary) BY bucket = BUCKET(hire_date, 20, \"1985-01-01T00:00:00Z\", \"1986-01-01T00:00:00Z\")\n| SORT bucket", "FROM employees\n| STATS s1 = b1 + 1, s2 = BUCKET(salary / 1000 + 999, 50.) + 2 BY b1 = BUCKET(salary / 100 + 99, 50.), b2 = BUCKET(salary / 1000 + 999, 50.)\n| SORT b1, b2\n| KEEP s1, b1, s2, b2", - "FROM employees \n| STATS dates = VALUES(birth_date) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count\n| LIMIT 3" + "FROM employees\n| STATS dates = MV_SORT(VALUES(birth_date)) BY b = BUCKET(birth_date + 1 HOUR, 1 YEAR) - 1 HOUR\n| EVAL d_count = MV_COUNT(dates)\n| SORT d_count, b\n| LIMIT 3" ], "preview" : false, "snapshot_only" : false diff --git a/docs/reference/esql/functions/kibana/definition/date_format.json b/docs/reference/esql/functions/kibana/definition/date_format.json index 6e2738fafb964..629415da30fa2 100644 --- a/docs/reference/esql/functions/kibana/definition/date_format.json +++ b/docs/reference/esql/functions/kibana/definition/date_format.json @@ -4,6 +4,18 @@ "name" : "date_format", "description" : "Returns a string representation of a date, in the provided format.", "signatures" : [ + { + "params" : [ + { + "name" : "dateFormat", + "type" : "date", + "optional" : true, + "description" : "Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json index d61534da81a6d..eb206cb9ddf4d 100644 --- a/docs/reference/esql/functions/kibana/definition/match.json +++ b/docs/reference/esql/functions/kibana/definition/match.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "eval", "name" : "match", - "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on fields from the text family like <> and <>,\nas well as other field types like keyword, boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/match_operator.json b/docs/reference/esql/functions/kibana/definition/match_operator.json index 44233bbddb653..c8cbf1cf9d966 100644 --- a/docs/reference/esql/functions/kibana/definition/match_operator.json +++ b/docs/reference/esql/functions/kibana/definition/match_operator.json @@ -2,7 +2,7 @@ "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.", "type" : "operator", "name" : "match_operator", - "description" : "Performs a <> on the specified field. Returns true if the provided query matches the row.", + "description" : "Use `MATCH` to perform a <> on the specified field.\nUsing `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL.\n\nMatch can be used on text fields, as well as other field types like boolean, dates, and numeric types.\n\nFor a simplified syntax, you can use the <> `:` operator instead of `MATCH`.\n\n`MATCH` returns true if the provided query matches the row.", "signatures" : [ { "params" : [ diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json index d9409bceb8e6f..210b9608f9eff 100644 --- a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json +++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json @@ -3,7 +3,7 @@ "type" : "eval", "name" : "to_date_nanos", "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).", - "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", + "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning.. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", "signatures" : [ { "params" : [ @@ -90,6 +90,6 @@ "returnType" : "date_nanos" } ], - "preview" : true, + "preview" : false, "snapshot_only" : false } diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md index 72258a1682936..80bf84351c188 100644 --- a/docs/reference/esql/functions/kibana/docs/match.md +++ b/docs/reference/esql/functions/kibana/docs/match.md @@ -6,7 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. -Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. +Match can be used on fields from the text family like <> and <>, +as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. diff --git a/docs/reference/esql/functions/kibana/docs/match_operator.md b/docs/reference/esql/functions/kibana/docs/match_operator.md index b0b6196798087..7681c2d1ce231 100644 --- a/docs/reference/esql/functions/kibana/docs/match_operator.md +++ b/docs/reference/esql/functions/kibana/docs/match_operator.md @@ -3,7 +3,14 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ --> ### MATCH_OPERATOR -Performs a <> on the specified field. Returns true if the provided query matches the row. +Use `MATCH` to perform a <> on the specified field. +Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. + +Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. + +For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. + +`MATCH` returns true if the provided query matches the row. ``` FROM books diff --git a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md index 0294802485ccb..1bce8d4fca832 100644 --- a/docs/reference/esql/functions/kibana/docs/to_date_nanos.md +++ b/docs/reference/esql/functions/kibana/docs/to_date_nanos.md @@ -5,4 +5,4 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ ### TO_DATE_NANOS Converts an input to a nanosecond-resolution date value (aka date_nanos). -Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. +Note: The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convertvalues outside of that range will result in null with a warning.. Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch. diff --git a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc index 977a0ac969e5d..2dfd13dac7e20 100644 --- a/docs/reference/esql/functions/layout/to_date_nanos.asciidoc +++ b/docs/reference/esql/functions/layout/to_date_nanos.asciidoc @@ -4,8 +4,6 @@ [[esql-to_date_nanos]] === `TO_DATE_NANOS` -preview::["Do not use on production environments. This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features."] - *Syntax* [.text-center] diff --git a/docs/reference/esql/functions/type-conversion-functions.asciidoc b/docs/reference/esql/functions/type-conversion-functions.asciidoc index 9ac9ec290c07b..bd70c2789dfa2 100644 --- a/docs/reference/esql/functions/type-conversion-functions.asciidoc +++ b/docs/reference/esql/functions/type-conversion-functions.asciidoc @@ -18,6 +18,7 @@ * <> * experimental:[] <> * <> +* <> * <> * <> * <> @@ -37,6 +38,7 @@ include::layout/to_cartesianpoint.asciidoc[] include::layout/to_cartesianshape.asciidoc[] include::layout/to_dateperiod.asciidoc[] include::layout/to_datetime.asciidoc[] +include::layout/to_date_nanos.asciidoc[] include::layout/to_degrees.asciidoc[] include::layout/to_double.asciidoc[] include::layout/to_geopoint.asciidoc[] diff --git a/docs/reference/esql/functions/types/date_format.asciidoc b/docs/reference/esql/functions/types/date_format.asciidoc index b2e97dfa8835a..580094e9be906 100644 --- a/docs/reference/esql/functions/types/date_format.asciidoc +++ b/docs/reference/esql/functions/types/date_format.asciidoc @@ -5,6 +5,7 @@ [%header.monospaced.styled,format=dsv,separator=|] |=== dateFormat | date | result +date | | keyword keyword | date | keyword text | date | keyword |=== diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index ca7de396147a8..b6b82422cbb4a 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -24,7 +24,6 @@ index settings, aliases, mappings, and index templates. * <> * <> * <> -* <> * <> * <> * <> @@ -143,6 +142,5 @@ include::indices/shrink-index.asciidoc[] include::indices/simulate-index.asciidoc[] include::indices/simulate-template.asciidoc[] include::indices/split-index.asciidoc[] -include::indices/apis/unfreeze.asciidoc[] include::indices/update-settings.asciidoc[] include::indices/put-mapping.asciidoc[] diff --git a/docs/reference/indices/apis/unfreeze.asciidoc b/docs/reference/indices/apis/unfreeze.asciidoc deleted file mode 100644 index 5d04d44db7443..0000000000000 --- a/docs/reference/indices/apis/unfreeze.asciidoc +++ /dev/null @@ -1,61 +0,0 @@ -[role="xpack"] -[[unfreeze-index-api]] -=== Unfreeze index API -++++ -Unfreeze index -++++ - -[WARNING] -.Deprecated in 7.14 -==== -In 8.0, we removed the ability to freeze an index. In previous versions, -freezing an index reduced its memory overhead. However, frozen indices are no -longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. -You can use this API to unfreeze indices that were frozen in 7.x. Frozen indices -are not related to the frozen data tier. -==== - -.New API reference -[sidebar] --- -For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs]. --- - -Unfreezes an index. - -[[unfreeze-index-api-request]] -==== {api-request-title} - -`POST //_unfreeze` - -[[unfreeze-index-api-prereqs]] -==== {api-prereq-title} - -* If the {es} {security-features} are enabled, you must have the `manage` -<> for the target index or index alias. - -[[unfreeze-index-api-desc]] -==== {api-description-title} - -When a frozen index is unfrozen, the index goes through the normal recovery -process and becomes writeable again. - -[[unfreeze-index-api-path-parms]] -==== {api-path-parms-title} - -``:: - (Required, string) Identifier for the index. - -[[unfreeze-index-api-examples]] -==== {api-examples-title} - -The following example unfreezes an index: - -[source,console] --------------------------------------------------- -POST /my-index-000001/_unfreeze --------------------------------------------------- -// TEST[s/^/PUT my-index-000001\n/] -// TEST[skip:unable to ignore deprecation warning] diff --git a/docs/reference/indices/index-mgmt.asciidoc b/docs/reference/indices/index-mgmt.asciidoc index 73643dbfd4b3b..131bc79faa40c 100644 --- a/docs/reference/indices/index-mgmt.asciidoc +++ b/docs/reference/indices/index-mgmt.asciidoc @@ -43,7 +43,7 @@ For more information on managing indices, refer to <>. * To filter the list of indices, use the search bar or click a badge. Badges indicate if an index is a <>, a -<>, or <>. +<>, or <>. * To drill down into the index <>, <>, and statistics, diff --git a/docs/reference/inference/chat-completion-inference.asciidoc b/docs/reference/inference/chat-completion-inference.asciidoc new file mode 100644 index 0000000000000..83a8f94634f2f --- /dev/null +++ b/docs/reference/inference/chat-completion-inference.asciidoc @@ -0,0 +1,417 @@ +[role="xpack"] +[[chat-completion-inference-api]] +=== Chat completion inference API + +Streams a chat completion response. + +IMPORTANT: The {infer} APIs enable you to use certain services, such as built-in {ml} models (ELSER, E5), models uploaded through Eland, Cohere, OpenAI, Azure, Google AI Studio, Google Vertex AI, Anthropic, Watsonx.ai, or Hugging Face. +For built-in models and models uploaded through Eland, the {infer} APIs offer an alternative way to use and manage trained models. +However, if you do not plan to use the {infer} APIs to use these models or if you want to use non-NLP models, use the <>. + + +[discrete] +[[chat-completion-inference-api-request]] +==== {api-request-title} + +`POST /_inference//_unified` + +`POST /_inference/chat_completion//_unified` + + +[discrete] +[[chat-completion-inference-api-prereqs]] +==== {api-prereq-title} + +* Requires the `monitor_inference` <> +(the built-in `inference_admin` and `inference_user` roles grant this privilege) +* You must use a client that supports streaming. + + +[discrete] +[[chat-completion-inference-api-desc]] +==== {api-description-title} + +The chat completion {infer} API enables real-time responses for chat completion tasks by delivering answers incrementally, reducing response times during computation. +It only works with the `chat_completion` task type for `openai` and `elastic` {infer} services. + +[NOTE] +==== +The `chat_completion` task type is only available within the _unified API and only supports streaming. +==== + +[discrete] +[[chat-completion-inference-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +The unique identifier of the {infer} endpoint. + + +``:: +(Optional, string) +The type of {infer} task that the model performs. If included, this must be set to the value `chat_completion`. + + +[discrete] +[[chat-completion-inference-api-request-body]] +==== {api-request-body-title} + +`messages`:: +(Required, array of objects) A list of objects representing the conversation. +Requests should generally only add new messages from the user (role `user`). The other message roles (`assistant`, `system`, or `tool`) should generally only be copied from the response to a previous completion request, such that the messages array is built up throughout a conversation. ++ +.Assistant message +[%collapsible%closed] +===== +`content`:: +(Required unless `tool_calls` is specified, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `assistant` for this type of message. ++ +`tool_calls`:: +(Optional, array of objects) +The tool calls generated by the model. ++ +.Examples +[%collapsible%closed] +====== +[source,js] +------------------------------------------------------------ +{ + "tool_calls": [ + { + "id": "call_KcAjWtAww20AihPHphUh46Gd", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Boston, MA\"}" + } + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== ++ +`id`::: +(Required, string) +The identifier of the tool call. ++ +`type`::: +(Required, string) +The type of tool call. This must be set to the value `function`. ++ +`function`::: +(Required, object) +The function that the model called. ++ +`name`:::: +(Required, string) +The name of the function to call. ++ +`arguments`:::: +(Required, string) +The arguments to call the function with in JSON format. +===== ++ +.System message +[%collapsible%closed] +===== +`content`::: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`::: +(Required, string) +The role of the message author. This should be set to `system` for this type of message. +===== ++ +.Tool message +[%collapsible%closed] +===== +`content`:: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `tool` for this type of message. ++ +`tool_call_id`:: +(Required, string) +The tool call that this message is responding to. +===== ++ +.User message +[%collapsible%closed] +===== +`content`:: +(Required, string or array of objects) +The contents of the message. ++ +include::inference-shared.asciidoc[tag=chat-completion-schema-content-with-examples] ++ +`role`:: +(Required, string) +The role of the message author. This should be set to `user` for this type of message. +===== + +`model`:: +(Optional, string) +The ID of the model to use. By default, the model ID is set to the value included when creating the inference endpoint. + +`max_completion_tokens`:: +(Optional, integer) +The upper bound limit for the number of tokens that can be generated for a completion request. + +`stop`:: +(Optional, array of strings) +A sequence of strings to control when the model should stop generating additional tokens. + +`temperature`:: +(Optional, float) +The sampling temperature to use. + +`tools`:: +(Optional, array of objects) +A list of tools that the model can call. ++ +.Structure +[%collapsible%closed] +===== +`type`:: +(Required, string) +The type of tool, must be set to the value `function`. ++ +`function`:: +(Required, object) +The function definition. ++ +`description`::: +(Optional, string) +A description of what the function does. This is used by the model to choose when and how to call the function. ++ +`name`::: +(Required, string) +The name of the function. ++ +`parameters`::: +(Optional, object) +The parameters the functional accepts. This should be formatted as a JSON object. ++ +`strict`::: +(Optional, boolean) +Whether to enable schema adherence when generating the function call. +===== ++ +.Examples +[%collapsible%closed] +====== +[source,js] +------------------------------------------------------------ +{ + "tools": [ + { + "type": "function", + "function": { + "name": "get_price_of_item", + "description": "Get the current price of an item", + "parameters": { + "type": "object", + "properties": { + "item": { + "id": "12345" + }, + "unit": { + "type": "currency" + } + } + } + } + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== + +`tool_choice`:: +(Optional, string or object) +Controls which tool is called by the model. ++ +String representation::: +One of `auto`, `none`, or `requrired`. `auto` allows the model to choose between calling tools and generating a message. `none` causes the model to not call any tools. `required` forces the model to call one or more tools. ++ +Object representation::: ++ +.Structure +[%collapsible%closed] +===== +`type`:: +(Required, string) +The type of the tool. This must be set to the value `function`. ++ +`function`:: +(Required, object) ++ +`name`::: +(Required, string) +The name of the function to call. +===== ++ +.Examples +[%collapsible%closed] +===== +[source,js] +------------------------------------------------------------ +{ + "tool_choice": { + "type": "function", + "function": { + "name": "get_current_weather" + } + } +} +------------------------------------------------------------ +// NOTCONSOLE +===== + +`top_p`:: +(Optional, float) +Nucleus sampling, an alternative to sampling with temperature. + +[discrete] +[[chat-completion-inference-api-example]] +==== {api-examples-title} + +The following example performs a chat completion on the example question with streaming. + + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": "What is Elastic?" + } + ] +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The following example performs a chat completion using an Assistant message with `tool_calls`. + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "messages": [ + { + "role": "assistant", + "content": "Let's find out what the weather is", + "tool_calls": [ <1> + { + "id": "call_KcAjWtAww20AihPHphUh46Gd", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\":\"Boston, MA\"}" + } + } + ] + }, + { <2> + "role": "tool", + "content": "The weather is cold", + "tool_call_id": "call_KcAjWtAww20AihPHphUh46Gd" + } + ] +} +------------------------------------------------------------ +// TEST[skip:TBD] + +<1> Each tool call needs a corresponding Tool message. +<2> The corresponding Tool message. + +The following example performs a chat completion using a User message with `tools` and `tool_choice`. + +[source,console] +------------------------------------------------------------ +POST _inference/chat_completion/openai-completion/_stream +{ + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What's the price of a scarf?" + } + ] + } + ], + "tools": [ + { + "type": "function", + "function": { + "name": "get_current_price", + "description": "Get the current price of a item", + "parameters": { + "type": "object", + "properties": { + "item": { + "id": "123" + } + } + } + } + } + ], + "tool_choice": { + "type": "function", + "function": { + "name": "get_current_price" + } + } +} +------------------------------------------------------------ +// TEST[skip:TBD] + +The API returns the following response when a request is made to the OpenAI service: + + +[source,txt] +------------------------------------------------------------ +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":"","role":"assistant"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":Elastic"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[{"delta":{"content":" is"},"index":0}],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk"}} + +(...) + +event: message +data: {"chat_completion":{"id":"chatcmpl-Ae0TWsy2VPnSfBbv5UztnSdYUMFP3","choices":[],"model":"gpt-4o-2024-08-06","object":"chat.completion.chunk","usage":{"completion_tokens":28,"prompt_tokens":16,"total_tokens":44}}} <1> + +event: message +data: [DONE] +------------------------------------------------------------ +// NOTCONSOLE + +<1> The last object message of the stream contains the token usage information. diff --git a/docs/reference/inference/inference-apis.asciidoc b/docs/reference/inference/inference-apis.asciidoc index ca273afc478ea..4f27409973ca2 100644 --- a/docs/reference/inference/inference-apis.asciidoc +++ b/docs/reference/inference/inference-apis.asciidoc @@ -26,6 +26,7 @@ the following APIs to manage {infer} models and perform {infer}: * <> * <> * <> +* <> * <> [[inference-landscape]] @@ -34,9 +35,9 @@ image::images/inference-landscape.jpg[A representation of the Elastic inference An {infer} endpoint enables you to use the corresponding {ml} model without manual deployment and apply it to your data at ingestion time through -<>. +<>. -Choose a model from your provider or use ELSER – a retrieval model trained by +Choose a model from your provider or use ELSER – a retrieval model trained by Elastic –, then create an {infer} endpoint by the <>. Now use <> to perform <> on your data. @@ -67,7 +68,7 @@ The following list contains the default {infer} endpoints listed by `inference_i Use the `inference_id` of the endpoint in a <> field definition or when creating an <>. The API call will automatically download and deploy the model which might take a couple of minutes. Default {infer} enpoints have {ml-docs}/ml-nlp-auto-scale.html#nlp-model-adaptive-allocations[adaptive allocations] enabled. -For these models, the minimum number of allocations is `0`. +For these models, the minimum number of allocations is `0`. If there is no {infer} activity that uses the endpoint, the number of allocations will scale down to `0` automatically after 15 minutes. @@ -84,7 +85,7 @@ Returning a long document in search results is less useful than providing the mo Each chunk will include the text subpassage and the corresponding embedding generated from it. By default, documents are split into sentences and grouped in sections up to 250 words with 1 sentence overlap so that each chunk shares a sentence with the previous chunk. -Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. +Overlapping ensures continuity and prevents vital contextual information in the input text from being lost by a hard break. {es} uses the https://unicode-org.github.io/icu-docs/[ICU4J] library to detect word and sentence boundaries for chunking. https://unicode-org.github.io/icu/userguide/boundaryanalysis/#word-boundary[Word boundaries] are identified by following a series of rules, not just the presence of a whitespace character. @@ -135,6 +136,7 @@ PUT _inference/sparse_embedding/small_chunk_size include::delete-inference.asciidoc[] include::get-inference.asciidoc[] include::post-inference.asciidoc[] +include::chat-completion-inference.asciidoc[] include::put-inference.asciidoc[] include::stream-inference.asciidoc[] include::update-inference.asciidoc[] diff --git a/docs/reference/inference/inference-shared.asciidoc b/docs/reference/inference/inference-shared.asciidoc index da497c6581e5d..b133c54082810 100644 --- a/docs/reference/inference/inference-shared.asciidoc +++ b/docs/reference/inference/inference-shared.asciidoc @@ -41,7 +41,7 @@ end::chunking-settings[] tag::chunking-settings-max-chunking-size[] Specifies the maximum size of a chunk in words. Defaults to `250`. -This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). +This value cannot be higher than `300` or lower than `20` (for `sentence` strategy) or `10` (for `word` strategy). end::chunking-settings-max-chunking-size[] tag::chunking-settings-overlap[] @@ -63,4 +63,48 @@ Specifies the chunking strategy. It could be either `sentence` or `word`. end::chunking-settings-strategy[] +tag::chat-completion-schema-content-with-examples[] +.Examples +[%collapsible%closed] +====== +String example +[source,js] +------------------------------------------------------------ +{ + "content": "Some string" +} +------------------------------------------------------------ +// NOTCONSOLE + +Object example +[source,js] +------------------------------------------------------------ +{ + "content": [ + { + "text": "Some text", + "type": "text" + } + ] +} +------------------------------------------------------------ +// NOTCONSOLE +====== + +String representation::: +(Required, string) +The text content. ++ +Object representation::: +`text`:::: +(Required, string) +The text content. ++ +`type`:::: +(Required, string) +This must be set to the value `text`. +end::chat-completion-schema-content-with-examples[] +tag::chat-completion-docs[] +For more information on how to use the `chat_completion` task type, please refer to the <>. +end::chat-completion-docs[] diff --git a/docs/reference/inference/put-inference.asciidoc b/docs/reference/inference/put-inference.asciidoc index f0c15323863d7..da07d1d3e7d84 100644 --- a/docs/reference/inference/put-inference.asciidoc +++ b/docs/reference/inference/put-inference.asciidoc @@ -42,7 +42,7 @@ include::inference-shared.asciidoc[tag=inference-id] include::inference-shared.asciidoc[tag=task-type] + -- -Refer to the service list in the <> for the available task types. +Refer to the service list in the <> for the available task types. -- @@ -61,7 +61,7 @@ The create {infer} API enables you to create an {infer} endpoint and configure a The following services are available through the {infer} API. -You can find the available task types next to the service name. +You can find the available task types next to the service name. Click the links to review the configuration details of the services: * <> (`completion`, `rerank`, `sparse_embedding`, `text_embedding`) @@ -73,10 +73,10 @@ Click the links to review the configuration details of the services: * <> (`rerank`, `sparse_embedding`, `text_embedding` - this service is for built-in models and models uploaded through Eland) * <> (`sparse_embedding`) * <> (`completion`, `text_embedding`) -* <> (`rerank`, `text_embedding`) +* <> (`rerank`, `text_embedding`) * <> (`text_embedding`) * <> (`text_embedding`) -* <> (`completion`, `text_embedding`) +* <> (`chat_completion`, `completion`, `text_embedding`) * <> (`text_embedding`) * <> (`text_embedding`, `rerank`) diff --git a/docs/reference/inference/service-openai.asciidoc b/docs/reference/inference/service-openai.asciidoc index e4be7f18e09dd..590f280b1c494 100644 --- a/docs/reference/inference/service-openai.asciidoc +++ b/docs/reference/inference/service-openai.asciidoc @@ -31,10 +31,18 @@ include::inference-shared.asciidoc[tag=task-type] -- Available task types: +* `chat_completion`, * `completion`, * `text_embedding`. -- +[NOTE] +==== +The `chat_completion` task type only supports streaming and only through the `_unified` API. + +include::inference-shared.asciidoc[tag=chat-completion-docs] +==== + [discrete] [[infer-service-openai-api-request-body]] ==== {api-request-body-title} @@ -61,7 +69,7 @@ include::inference-shared.asciidoc[tag=chunking-settings-strategy] `service`:: (Required, string) -The type of service supported for the specified task type. In this case, +The type of service supported for the specified task type. In this case, `openai`. `service_settings`:: @@ -176,4 +184,4 @@ PUT _inference/completion/openai-completion } } ------------------------------------------------------------ -// TEST[skip:TBD] \ No newline at end of file +// TEST[skip:TBD] diff --git a/docs/reference/inference/stream-inference.asciidoc b/docs/reference/inference/stream-inference.asciidoc index 42abb589f9afd..4a3ce31909712 100644 --- a/docs/reference/inference/stream-inference.asciidoc +++ b/docs/reference/inference/stream-inference.asciidoc @@ -38,8 +38,12 @@ However, if you do not plan to use the {infer} APIs to use these models or if yo ==== {api-description-title} The stream {infer} API enables real-time responses for completion tasks by delivering answers incrementally, reducing response times during computation. -It only works with the `completion` task type. +It only works with the `completion` and `chat_completion` task types. +[NOTE] +==== +include::inference-shared.asciidoc[tag=chat-completion-docs] +==== [discrete] [[stream-inference-api-path-params]] diff --git a/docs/reference/query-dsl/bool-query.asciidoc b/docs/reference/query-dsl/bool-query.asciidoc index c24135a370914..27220f0d85149 100644 --- a/docs/reference/query-dsl/bool-query.asciidoc +++ b/docs/reference/query-dsl/bool-query.asciidoc @@ -13,21 +13,24 @@ occurrence types are: |======================================================================= |Occur |Description |`must` |The clause (query) must appear in matching documents and will -contribute to the score. +contribute to the score. Each query defined under a `must` acts as a logical "AND", returning only documents that match _all_ the specified queries. + +|`should` |The clause (query) should appear in the matching document. Each query defined under a `should` acts as a logical "OR", returning documents that match _any_ of the specified queries. |`filter` |The clause (query) must appear in matching documents. However unlike `must` the score of the query will be ignored. Filter clauses are executed in <>, meaning that scoring is ignored -and clauses are considered for caching. - -|`should` |The clause (query) should appear in the matching document. +and clauses are considered for caching. Each query defined under a `filter` acts as a logical "AND", returning only documents that match _all_ the specified queries. |`must_not` |The clause (query) must not appear in the matching documents. Clauses are executed in <> meaning that scoring is ignored and clauses are considered for caching. Because scoring is -ignored, a score of `0` for all documents is returned. +ignored, a score of `0` for all documents is returned. Each query defined under a `must_not` acts as a logical "NOT", returning only documents that do not match any of the specified queries. + |======================================================================= +The `must` and `should` clauses function as logical AND, OR operators, contributing to the scoring of results. However, these results will not be cached for faster retrieval. In contrast, the `filter` and `must_not` clauses are used to include or exclude results without impacting the score, unless used within a `constant_score` query. + The `bool` query takes a _more-matches-is-better_ approach, so the score from each matching `must` or `should` clause will be added together to provide the final `_score` for each document. diff --git a/docs/reference/redirects.asciidoc b/docs/reference/redirects.asciidoc index c3bf84fa600d2..9c0f0092214ed 100644 --- a/docs/reference/redirects.asciidoc +++ b/docs/reference/redirects.asciidoc @@ -156,10 +156,16 @@ See <>. The freeze index API was removed in 8.0. // tag::frozen-removal-explanation[] Frozen indices are no longer useful due to -https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[recent -improvements in heap memory usage]. +https://www.elastic.co/blog/significantly-decrease-your-elasticsearch-heap-memory-usage[improvements +in heap memory usage]. // end::frozen-removal-explanation[] +[role="exclude",id="unfreeze-index-api"] +=== Unfreeze index API + +The unfreeze index API was removed in 9.0. +include::redirects.asciidoc[tag=frozen-removal-explanation] + [role="exclude",id="ilm-freeze"] === Freeze {ilm-init} action @@ -1749,8 +1755,10 @@ See <>. === Frozen indices // tag::frozen-index-redirect[] - -For API documentation, see <>. +Older versions of {es} provided the option to reduce the amount of data kept in memory for an index, at the expense of +increasing search latency. This was known as 'freezing' the index. +include::redirects.asciidoc[tag=frozen-removal-explanation] +The freeze index API was removed in 8.0, and the unfreeze index API was removed in 9.0. // end::frozen-index-redirect[] [role="exclude",id="best_practices"] diff --git a/docs/reference/sql/language/indices.asciidoc b/docs/reference/sql/language/indices.asciidoc index 1dee7f0840ade..1912a020ab0be 100644 --- a/docs/reference/sql/language/indices.asciidoc +++ b/docs/reference/sql/language/indices.asciidoc @@ -100,7 +100,7 @@ requires the keyword `LIKE` for SQL `LIKE` pattern. [[sql-index-frozen]] === Frozen Indices -By default, {es-sql} doesn't search <>. To +By default, {es-sql} doesn't search <>. To search frozen indices, use one of the following features: dedicated configuration parameter:: diff --git a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java index 3a359eb921fc8..69fc57973f68a 100644 --- a/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java +++ b/libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java @@ -13,10 +13,16 @@ import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -189,4 +195,28 @@ public interface EntitlementChecker { // The only implementation of SSLSession#getSessionContext(); unfortunately it's an interface, so we need to check the implementation void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession); + + void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr); + + void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr); + + void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p); + + void check$java_net_DatagramSocket$joinGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_DatagramSocket$leaveGroup(Class callerClass, DatagramSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, InetAddress addr); + + void check$java_net_MulticastSocket$leaveGroup(Class callerClass, MulticastSocket that, SocketAddress addr, NetworkInterface ni); + + void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl); } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java index 6dbb684c71514..fae873123528d 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/DummyImplementations.java @@ -9,8 +9,15 @@ package org.elasticsearch.entitlement.qa.common; +import java.io.IOException; +import java.net.DatagramPacket; +import java.net.DatagramSocket; +import java.net.DatagramSocketImpl; import java.net.InetAddress; +import java.net.NetworkInterface; import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; import java.security.cert.Certificate; import java.text.BreakIterator; import java.text.Collator; @@ -327,8 +334,77 @@ public Socket createSocket(Socket s, String host, int port, boolean autoClose) { } } + static class DummyDatagramSocket extends DatagramSocket { + DummyDatagramSocket() throws SocketException { + super(new DatagramSocketImpl() { + @Override + protected void create() throws SocketException {} + + @Override + protected void bind(int lport, InetAddress laddr) throws SocketException {} + + @Override + protected void send(DatagramPacket p) throws IOException {} + + @Override + protected int peek(InetAddress i) throws IOException { + return 0; + } + + @Override + protected int peekData(DatagramPacket p) throws IOException { + return 0; + } + + @Override + protected void receive(DatagramPacket p) throws IOException {} + + @Override + protected void setTTL(byte ttl) throws IOException {} + + @Override + protected byte getTTL() throws IOException { + return 0; + } + + @Override + protected void setTimeToLive(int ttl) throws IOException {} + + @Override + protected int getTimeToLive() throws IOException { + return 0; + } + + @Override + protected void join(InetAddress inetaddr) throws IOException {} + + @Override + protected void leave(InetAddress inetaddr) throws IOException {} + + @Override + protected void joinGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void leaveGroup(SocketAddress mcastaddr, NetworkInterface netIf) throws IOException {} + + @Override + protected void close() {} + + @Override + public void setOption(int optID, Object value) throws SocketException {} + + @Override + public Object getOption(int optID) throws SocketException { + return null; + } + + @Override + protected void connect(InetAddress address, int port) throws SocketException {} + }); + } + } + private static RuntimeException unexpected() { return new IllegalStateException("This method isn't supposed to be called"); } - } diff --git a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java index 1dd8daf556226..3a5480f468528 100644 --- a/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java +++ b/libs/entitlement/qa/common/src/main/java/org/elasticsearch/entitlement/qa/common/RestEntitlementsCheckAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyBreakIteratorProvider; import org.elasticsearch.entitlement.qa.common.DummyImplementations.DummyCalendarDataProvider; @@ -32,14 +33,18 @@ import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.io.UncheckedIOException; +import java.net.DatagramPacket; import java.net.DatagramSocket; import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.MalformedURLException; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; import java.net.ServerSocket; import java.net.Socket; +import java.net.SocketException; import java.net.URL; import java.net.URLClassLoader; import java.net.URLConnection; @@ -71,20 +76,20 @@ public class RestEntitlementsCheckAction extends BaseRestHandler { public static final Thread NO_OP_SHUTDOWN_HOOK = new Thread(() -> {}, "Shutdown hook for testing"); private final String prefix; - record CheckAction(Runnable action, boolean isAlwaysDeniedToPlugins) { + record CheckAction(CheckedRunnable action, boolean isAlwaysDeniedToPlugins) { /** * These cannot be granted to plugins, so our test plugins cannot test the "allowed" case. * Used both for always-denied entitlements as well as those granted only to the server itself. */ - static CheckAction deniedToPlugins(Runnable action) { + static CheckAction deniedToPlugins(CheckedRunnable action) { return new CheckAction(action, true); } - static CheckAction forPlugins(Runnable action) { + static CheckAction forPlugins(CheckedRunnable action) { return new CheckAction(action, false); } - static CheckAction alwaysDenied(Runnable action) { + static CheckAction alwaysDenied(CheckedRunnable action) { return new CheckAction(action, true); } } @@ -142,7 +147,13 @@ static CheckAction alwaysDenied(Runnable action) { entry("createURLStreamHandlerProvider", alwaysDenied(RestEntitlementsCheckAction::createURLStreamHandlerProvider)), entry("createURLWithURLStreamHandler", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler)), entry("createURLWithURLStreamHandler2", alwaysDenied(RestEntitlementsCheckAction::createURLWithURLStreamHandler2)), - entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)) + entry("sslSessionImpl_getSessionContext", alwaysDenied(RestEntitlementsCheckAction::sslSessionImplGetSessionContext)), + entry("datagram_socket_bind", forPlugins(RestEntitlementsCheckAction::bindDatagramSocket)), + entry("datagram_socket_connect", forPlugins(RestEntitlementsCheckAction::connectDatagramSocket)), + entry("datagram_socket_send", forPlugins(RestEntitlementsCheckAction::sendDatagramSocket)), + entry("datagram_socket_receive", forPlugins(RestEntitlementsCheckAction::receiveDatagramSocket)), + entry("datagram_socket_join_group", forPlugins(RestEntitlementsCheckAction::joinGroupDatagramSocket)), + entry("datagram_socket_leave_group", forPlugins(RestEntitlementsCheckAction::leaveGroupDatagramSocket)) ); private static void createURLStreamHandlerProvider() { @@ -154,43 +165,33 @@ public URLStreamHandler createURLStreamHandler(String protocol) { }; } - private static void sslSessionImplGetSessionContext() { + private static void sslSessionImplGetSessionContext() throws IOException { SSLSocketFactory factory = HttpsURLConnection.getDefaultSSLSocketFactory(); try (SSLSocket socket = (SSLSocket) factory.createSocket()) { SSLSession session = socket.getSession(); session.getSessionContext(); - } catch (IOException e) { - throw new RuntimeException(e); } } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler() { - try { - var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler() throws MalformedURLException { + var x = new URL("http", "host", 1234, "file", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } @SuppressWarnings("deprecation") - private static void createURLWithURLStreamHandler2() { - try { - var x = new URL(null, "spec", new URLStreamHandler() { - @Override - protected URLConnection openConnection(URL u) { - return null; - } - }); - } catch (MalformedURLException e) { - throw new RuntimeException(e); - } + private static void createURLWithURLStreamHandler2() throws MalformedURLException { + var x = new URL(null, "spec", new URLStreamHandler() { + @Override + protected URLConnection openConnection(URL u) { + return null; + } + }); } private static void createInetAddressResolverProvider() { @@ -215,12 +216,8 @@ private static void setDefaultProxySelector() { ProxySelector.setDefault(null); } - private static void setDefaultSSLContext() { - try { - SSLContext.setDefault(SSLContext.getDefault()); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } + private static void setDefaultSSLContext() throws NoSuchAlgorithmException { + SSLContext.setDefault(SSLContext.getDefault()); } private static void setDefaultHostnameVerifier() { @@ -246,28 +243,18 @@ private static void systemExit() { System.exit(123); } - private static void createClassLoader() { + private static void createClassLoader() throws IOException { try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) { logger.info("Created URLClassLoader [{}]", classLoader.getName()); - } catch (IOException e) { - throw new UncheckedIOException(e); } } - private static void processBuilder_start() { - try { - new ProcessBuilder("").start(); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_start() throws IOException { + new ProcessBuilder("").start(); } - private static void processBuilder_startPipeline() { - try { - ProcessBuilder.startPipeline(List.of()); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void processBuilder_startPipeline() throws IOException { + ProcessBuilder.startPipeline(List.of()); } private static void setHttpsConnectionProperties() { @@ -355,12 +342,8 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void datagramSocket$$setDatagramSocketImplFactory() { - try { - DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void datagramSocket$$setDatagramSocketImplFactory() throws IOException { + DatagramSocket.setDatagramSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void httpURLConnection$$setFollowRedirects() { @@ -369,22 +352,14 @@ private static void setHttpsConnectionProperties() { @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void serverSocket$$setSocketFactory() { - try { - ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void serverSocket$$setSocketFactory() throws IOException { + ServerSocket.setSocketFactory(() -> { throw new IllegalStateException(); }); } @SuppressWarnings("deprecation") @SuppressForbidden(reason = "We're required to prevent calls to this forbidden API") - private static void socket$$setSocketImplFactory() { - try { - Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); - } catch (IOException e) { - throw new IllegalStateException(e); - } + private static void socket$$setSocketImplFactory() throws IOException { + Socket.setSocketImplFactory(() -> { throw new IllegalStateException(); }); } private static void url$$setURLStreamHandlerFactory() { @@ -399,6 +374,51 @@ private static void setHttpsConnectionProperties() { URLConnection.setContentHandlerFactory(__ -> { throw new IllegalStateException(); }); } + private static void bindDatagramSocket() throws SocketException { + try (var socket = new DatagramSocket(null)) { + socket.bind(null); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void connectDatagramSocket() throws SocketException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.connect(new InetSocketAddress(1234)); + } + } + + private static void joinGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.joinGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + private static void leaveGroupDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.leaveGroup( + new InetSocketAddress(InetAddress.getByAddress(new byte[] { (byte) 230, 0, 0, 1 }), 1234), + NetworkInterface.getByIndex(0) + ); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void sendDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.send(new DatagramPacket(new byte[] { 0 }, 1, InetAddress.getLocalHost(), 1234)); + } + } + + @SuppressForbidden(reason = "testing entitlements") + private static void receiveDatagramSocket() throws IOException { + try (var socket = new DummyImplementations.DummyDatagramSocket()) { + socket.receive(new DatagramPacket(new byte[1], 1, InetAddress.getLocalHost(), 1234)); + } + } + public RestEntitlementsCheckAction(String prefix) { this.prefix = prefix; } diff --git a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml index 30fc9f0abeec0..05a94f09264a8 100644 --- a/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed-nonmodular/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ ALL-UNNAMED: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml index 0a25570a9f624..0d2c66c2daa2c 100644 --- a/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml +++ b/libs/entitlement/qa/entitlement-allowed/src/main/plugin-metadata/entitlement-policy.yaml @@ -1,3 +1,8 @@ org.elasticsearch.entitlement.qa.common: - create_class_loader - set_https_connection_properties + - network: + actions: + - listen + - accept + - connect diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java index ca4aaceabcebf..dd39ec3c5fe43 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java @@ -10,16 +10,23 @@ package org.elasticsearch.entitlement.runtime.api; import org.elasticsearch.entitlement.bridge.EntitlementChecker; +import org.elasticsearch.entitlement.runtime.policy.NetworkEntitlement; import org.elasticsearch.entitlement.runtime.policy.PolicyManager; import java.io.InputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.net.ContentHandlerFactory; +import java.net.DatagramPacket; +import java.net.DatagramSocket; import java.net.DatagramSocketImplFactory; import java.net.FileNameMap; +import java.net.InetAddress; +import java.net.MulticastSocket; +import java.net.NetworkInterface; import java.net.ProxySelector; import java.net.ResponseCache; +import java.net.SocketAddress; import java.net.SocketImplFactory; import java.net.URL; import java.net.URLStreamHandler; @@ -349,4 +356,68 @@ public ElasticsearchEntitlementChecker(PolicyManager policyManager) { public void check$sun_security_ssl_SSLSessionImpl$getSessionContext(Class callerClass, SSLSession sslSession) { policyManager.checkReadSensitiveNetworkInformation(callerClass); } + + @Override + public void check$java_net_DatagramSocket$bind(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.LISTEN_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$connect(Class callerClass, DatagramSocket that, SocketAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$send(Class callerClass, DatagramSocket that, DatagramPacket p) { + var actions = NetworkEntitlement.CONNECT_ACTION; + if (p.getAddress().isMulticastAddress()) { + actions |= NetworkEntitlement.ACCEPT_ACTION; + } + policyManager.checkNetworkAccess(callerClass, actions); + } + + @Override + public void check$java_net_DatagramSocket$receive(Class callerClass, DatagramSocket that, DatagramPacket p) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$joinGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_DatagramSocket$leaveGroup(Class caller, DatagramSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class callerClass, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$joinGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, InetAddress addr) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$leaveGroup(Class caller, MulticastSocket that, SocketAddress addr, NetworkInterface ni) { + policyManager.checkNetworkAccess(caller, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } + + @Override + public void check$java_net_MulticastSocket$send(Class callerClass, MulticastSocket that, DatagramPacket p, byte ttl) { + policyManager.checkNetworkAccess(callerClass, NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION); + } } diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java new file mode 100644 index 0000000000000..b6c6a41d5be7f --- /dev/null +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlement.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.core.Strings; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.StringJoiner; + +import static java.util.Map.entry; + +/** + * Describes a network entitlement (sockets) with actions. + */ +public class NetworkEntitlement implements Entitlement { + + public static final int LISTEN_ACTION = 0x1; + public static final int CONNECT_ACTION = 0x2; + public static final int ACCEPT_ACTION = 0x4; + + static final String LISTEN = "listen"; + static final String CONNECT = "connect"; + static final String ACCEPT = "accept"; + + private static final Map ACTION_MAP = Map.ofEntries( + entry(LISTEN, LISTEN_ACTION), + entry(CONNECT, CONNECT_ACTION), + entry(ACCEPT, ACCEPT_ACTION) + ); + + private final int actions; + + @ExternalEntitlement(parameterNames = { "actions" }, esModulesOnly = false) + public NetworkEntitlement(List actionsList) { + + int actionsInt = 0; + + for (String actionString : actionsList) { + var action = ACTION_MAP.get(actionString); + if (action == null) { + throw new IllegalArgumentException("unknown network action [" + actionString + "]"); + } + if ((actionsInt & action) == action) { + throw new IllegalArgumentException(Strings.format("network action [%s] specified multiple times", actionString)); + } + actionsInt |= action; + } + + this.actions = actionsInt; + } + + public static Object printActions(int actions) { + var joiner = new StringJoiner(","); + for (var entry : ACTION_MAP.entrySet()) { + var action = entry.getValue(); + if ((actions & action) == action) { + joiner.add(entry.getKey()); + } + } + return joiner.toString(); + } + + /** + * For the actions to match, the actions present in this entitlement must be a superset + * of the actions required by a check. + * There is only one "negative" case (action required by the check but not present in the entitlement), + * and it can be expressed efficiently via this truth table: + * this.actions | requiredActions | + * 0 | 0 | 0 + * 0 | 1 | 1 --> NOT this.action AND requiredActions + * 1 | 0 | 0 + * 1 | 1 | 0 + * + * @param requiredActions the actions required to be present for a check to pass + * @return true if requiredActions are present, false otherwise + */ + public boolean matchActions(int requiredActions) { + return (~this.actions & requiredActions) == 0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + NetworkEntitlement that = (NetworkEntitlement) o; + return actions == that.actions; + } + + @Override + public int hashCode() { + return Objects.hash(actions); + } + + @Override + public String toString() { + return "NetworkEntitlement{actions=" + actions + '}'; + } +} diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java index 57449a23a8215..f039fbda3dfbd 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java @@ -52,7 +52,11 @@ public boolean hasEntitlement(Class entitlementClass) { } public Stream getEntitlements(Class entitlementClass) { - return entitlementsByType.get(entitlementClass).stream().map(entitlementClass::cast); + var entitlements = entitlementsByType.get(entitlementClass); + if (entitlements == null) { + return Stream.empty(); + } + return entitlements.stream().map(entitlementClass::cast); } } @@ -190,6 +194,34 @@ private String operationDescription(String methodName) { return methodName.substring(methodName.indexOf('$')); } + public void checkNetworkAccess(Class callerClass, int actions) { + var requestingClass = requestingClass(callerClass); + if (isTriviallyAllowed(requestingClass)) { + return; + } + + ModuleEntitlements entitlements = getEntitlements(requestingClass); + if (entitlements.getEntitlements(NetworkEntitlement.class).anyMatch(n -> n.matchActions(actions))) { + logger.debug( + () -> Strings.format( + "Entitled: class [%s], module [%s], entitlement [Network], actions [Ox%X]", + requestingClass, + requestingClass.getModule().getName(), + actions + ) + ); + return; + } + throw new NotEntitledException( + Strings.format( + "Missing entitlement: class [%s], module [%s], entitlement [Network], actions [%s]", + requestingClass, + requestingClass.getModule().getName(), + NetworkEntitlement.printActions(actions) + ) + ); + } + private void checkEntitlementPresent(Class callerClass, Class entitlementClass) { var requestingClass = requestingClass(callerClass); if (isTriviallyAllowed(requestingClass)) { diff --git a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java index 013acf8f22fae..ac4d4afdd97f8 100644 --- a/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java +++ b/libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyParser.java @@ -37,7 +37,8 @@ public class PolicyParser { private static final Map> EXTERNAL_ENTITLEMENTS = Stream.of( FileEntitlement.class, CreateClassLoaderEntitlement.class, - SetHttpsConnectionPropertiesEntitlement.class + SetHttpsConnectionPropertiesEntitlement.class, + NetworkEntitlement.class ).collect(Collectors.toUnmodifiableMap(PolicyParser::getEntitlementTypeName, Function.identity())); protected final XContentParser policyParser; diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java new file mode 100644 index 0000000000000..91051d48c365f --- /dev/null +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/NetworkEntitlementTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.entitlement.runtime.policy; + +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.is; + +public class NetworkEntitlementTests extends ESTestCase { + + public void testMatchesActions() { + var listenEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.LISTEN)); + var emptyEntitlement = new NetworkEntitlement(List.of()); + var connectAcceptEntitlement = new NetworkEntitlement(List.of(NetworkEntitlement.CONNECT, NetworkEntitlement.ACCEPT)); + + assertThat(listenEntitlement.matchActions(0), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(true)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(listenEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + + assertThat(connectAcceptEntitlement.matchActions(0), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(true)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(connectAcceptEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(true)); + + assertThat(emptyEntitlement.matchActions(0), is(true)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.LISTEN_ACTION | NetworkEntitlement.CONNECT_ACTION), is(false)); + assertThat(emptyEntitlement.matchActions(NetworkEntitlement.CONNECT_ACTION | NetworkEntitlement.ACCEPT_ACTION), is(false)); + } +} diff --git a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java index 4d17fc92e1578..1e0c31d2280b8 100644 --- a/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java +++ b/libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java @@ -52,6 +52,22 @@ public void testPolicyBuilderOnExternalPlugin() throws IOException { assertEquals(expected, parsedPolicy); } + public void testParseNetwork() throws IOException { + Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" + entitlement-module-name: + - network: + actions: + - listen + - accept + - connect + """.getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy(); + Policy expected = new Policy( + "test-policy.yaml", + List.of(new Scope("entitlement-module-name", List.of(new NetworkEntitlement(List.of("listen", "accept", "connect"))))) + ); + assertEquals(expected, parsedPolicy); + } + public void testParseCreateClassloader() throws IOException { Policy parsedPolicy = new PolicyParser(new ByteArrayInputStream(""" entitlement-module-name: diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java index 339a4ec24ca13..43447cfa21a62 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/APM.java @@ -92,14 +92,7 @@ public List> getSettings() { APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING, APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES, - // The settings below are deprecated and are currently kept as fallback. - APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING, - APMAgentSettings.TRACING_APM_API_KEY_SETTING, - APMAgentSettings.TRACING_APM_ENABLED_SETTING, - APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING, - APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING, - APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES + APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES ); } } diff --git a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java index f66683a787bc0..8647761e2defe 100644 --- a/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java +++ b/modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java @@ -25,9 +25,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; -import java.util.function.Function; -import static org.elasticsearch.common.settings.Setting.Property.Deprecated; import static org.elasticsearch.common.settings.Setting.Property.NodeScope; import static org.elasticsearch.common.settings.Setting.Property.OperatorDynamic; @@ -101,9 +99,6 @@ public void setAgentSetting(String key, String value) { private static final String TELEMETRY_SETTING_PREFIX = "telemetry."; - // The old legacy prefix - private static final String LEGACY_TRACING_APM_SETTING_PREFIX = "tracing.apm."; - /** * Allow-list of APM agent config keys users are permitted to configure. * @see APM Java Agent Configuration @@ -248,56 +243,24 @@ private static Setting concreteAgentSetting(String namespace, String qua public static final Setting.AffixSetting APM_AGENT_SETTINGS = Setting.prefixKeySetting( TELEMETRY_SETTING_PREFIX + "agent.", - LEGACY_TRACING_APM_SETTING_PREFIX + "agent.", - (namespace, qualifiedKey) -> qualifiedKey.startsWith(LEGACY_TRACING_APM_SETTING_PREFIX) - ? concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic, Deprecated) - : concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) + null, // no fallback + (namespace, qualifiedKey) -> concreteAgentSetting(namespace, qualifiedKey, NodeScope, OperatorDynamic) ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_INCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_INCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.include", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_INCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.include", - TRACING_APM_NAMES_INCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING. - */ - @Deprecated - public static final Setting> TRACING_APM_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "names.exclude", - OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.listSetting( + public static final Setting> TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING = Setting.stringListSetting( TELEMETRY_SETTING_PREFIX + "tracing.names.exclude", - TRACING_APM_NAMES_EXCLUDE_SETTING, - Function.identity(), OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_SANITIZE_FIELD_NAMES. - */ - @Deprecated - public static final Setting> TRACING_APM_SANITIZE_FIELD_NAMES = Setting.stringListSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "sanitize_field_names", + public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.stringListSetting( + TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", List.of( "password", "passwd", @@ -313,33 +276,12 @@ private static Setting concreteAgentSetting(String namespace, String qua "set-cookie" ), OperatorDynamic, - NodeScope, - Deprecated - ); - - public static final Setting> TELEMETRY_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting( - TELEMETRY_SETTING_PREFIX + "tracing.sanitize_field_names", - TRACING_APM_SANITIZE_FIELD_NAMES, - Function.identity(), - OperatorDynamic, NodeScope ); - /** - * @deprecated in favor of TELEMETRY_TRACING_ENABLED_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_ENABLED_SETTING = Setting.boolSetting( - LEGACY_TRACING_APM_SETTING_PREFIX + "enabled", - false, - OperatorDynamic, - NodeScope, - Deprecated - ); - public static final Setting TELEMETRY_TRACING_ENABLED_SETTING = Setting.boolSetting( TELEMETRY_SETTING_PREFIX + "tracing.enabled", - TRACING_APM_ENABLED_SETTING, + false, OperatorDynamic, NodeScope ); @@ -351,33 +293,13 @@ private static Setting concreteAgentSetting(String namespace, String qua NodeScope ); - /** - * @deprecated in favor of TELEMETRY_SECRET_TOKEN_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_SECRET_TOKEN_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "secret_token", - null, - Deprecated - ); - public static final Setting TELEMETRY_SECRET_TOKEN_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "secret_token", - TRACING_APM_SECRET_TOKEN_SETTING - ); - - /** - * @deprecated in favor of TELEMETRY_API_KEY_SETTING. - */ - @Deprecated - public static final Setting TRACING_APM_API_KEY_SETTING = SecureSetting.secureString( - LEGACY_TRACING_APM_SETTING_PREFIX + "api_key", - null, - Deprecated + null ); public static final Setting TELEMETRY_API_KEY_SETTING = SecureSetting.secureString( TELEMETRY_SETTING_PREFIX + "api_key", - TRACING_APM_API_KEY_SETTING + null ); } diff --git a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java index a60048c82a3c9..5516672420924 100644 --- a/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java +++ b/modules/apm/src/test/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettingsTests.java @@ -11,8 +11,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.MockSecureSettings; -import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; import org.mockito.Mockito; @@ -21,21 +19,13 @@ import java.util.Set; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.APM_AGENT_SETTINGS; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_API_KEY_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_METRICS_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_SECRET_TOKEN_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_ENABLED_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_NAMES_INCLUDE_SETTING; import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TELEMETRY_TRACING_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_API_KEY_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_ENABLED_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_EXCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_NAMES_INCLUDE_SETTING; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SANITIZE_FIELD_NAMES; -import static org.elasticsearch.telemetry.apm.internal.APMAgentSettings.TRACING_APM_SECRET_TOKEN_SETTING; -import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasItem; import static org.mockito.Mockito.clearInvocations; import static org.mockito.Mockito.mock; @@ -70,14 +60,6 @@ public void testEnableTracing() { } } - public void testEnableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), true).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testEnableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -121,14 +103,6 @@ public void testDisableTracing() { } } - public void testDisableTracingUsingLegacySetting() { - Settings settings = Settings.builder().put(TRACING_APM_ENABLED_SETTING.getKey(), false).build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "false"); - assertWarnings("[tracing.apm.enabled] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - public void testDisableMetrics() { for (boolean tracingEnabled : List.of(true, false)) { clearInvocations(apmAgentSettings, apmTelemetryProvider.getMeterService()); @@ -181,70 +155,18 @@ public void testSetAgentSettings() { verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); } - public void testSetAgentsSettingsWithLegacyPrefix() { - Settings settings = Settings.builder() - .put(TELEMETRY_TRACING_ENABLED_SETTING.getKey(), true) - .put("tracing.apm.agent.span_compression_enabled", "true") - .build(); - apmAgentSettings.initAgentSystemProperties(settings); - - verify(apmAgentSettings).setAgentSetting("recording", "true"); - verify(apmAgentSettings).setAgentSetting("span_compression_enabled", "true"); - assertWarnings( - "[tracing.apm.agent.span_compression_enabled] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - /** * Check that invalid or forbidden APM agent settings are rejected. */ public void testRejectForbiddenOrUnknownAgentSettings() { - List prefixes = List.of(APM_AGENT_SETTINGS.getKey(), "tracing.apm.agent."); - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); - Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); - assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - } - // though, accept / ignore nested global_labels - for (String prefix : prefixes) { - Settings settings = Settings.builder().put(prefix + "global_labels.abc", "123").build(); - APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(settings); - - if (prefix.startsWith("tracing.apm.agent.")) { - assertWarnings( - "[tracing.apm.agent.global_labels.abc] setting was deprecated in Elasticsearch and will be removed in a future release." - ); - } - } - } - - public void testTelemetryTracingNamesIncludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_INCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_INCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.include] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingNamesExcludeFallback() { - Settings settings = Settings.builder().put(TRACING_APM_NAMES_EXCLUDE_SETTING.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_NAMES_EXCLUDE_SETTING.get(settings); - - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings("[tracing.apm.names.exclude] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryTracingSanitizeFieldNamesFallback() { - Settings settings = Settings.builder().put(TRACING_APM_SANITIZE_FIELD_NAMES.getKey(), "abc,xyz").build(); - - List included = TELEMETRY_TRACING_SANITIZE_FIELD_NAMES.get(settings); + String prefix = APM_AGENT_SETTINGS.getKey(); + Settings settings = Settings.builder().put(prefix + "unknown", "true").build(); + Exception exception = expectThrows(IllegalArgumentException.class, () -> APM_AGENT_SETTINGS.getAsMap(settings)); + assertThat(exception.getMessage(), containsString("[" + prefix + "unknown]")); - assertThat(included, containsInAnyOrder("abc", "xyz")); - assertWarnings( - "[tracing.apm.sanitize_field_names] setting was deprecated in Elasticsearch and will be removed in a future release." - ); + // though, accept / ignore nested global_labels + var map = APMAgentSettings.APM_AGENT_SETTINGS.getAsMap(Settings.builder().put(prefix + "global_labels.abc", "123").build()); + assertThat(map, hasEntry("global_labels.abc", "123")); } public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { @@ -252,28 +174,6 @@ public void testTelemetryTracingSanitizeFieldNamesFallbackDefault() { assertThat(included, hasItem("password")); // and more defaults } - public void testTelemetrySecretTokenFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_SECRET_TOKEN_SETTING.getKey(), "verysecret"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_SECRET_TOKEN_SETTING.get(settings)) { - assertEquals("verysecret", secureString.toString()); - } - assertWarnings("[tracing.apm.secret_token] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - - public void testTelemetryApiKeyFallback() { - MockSecureSettings secureSettings = new MockSecureSettings(); - secureSettings.setString(TRACING_APM_API_KEY_SETTING.getKey(), "abc"); - Settings settings = Settings.builder().setSecureSettings(secureSettings).build(); - - try (SecureString secureString = TELEMETRY_API_KEY_SETTING.get(settings)) { - assertEquals("abc", secureString.toString()); - } - assertWarnings("[tracing.apm.api_key] setting was deprecated in Elasticsearch and will be removed in a future release."); - } - /** * Check that invalid or forbidden APM agent settings are rejected if their last part resembles an allowed setting. */ diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 4a3dfac36d4ec..2739eb51376ea 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -596,7 +596,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(dataStreamName, indicesAdmin().prepareGetFieldMappings(dataStreamName), false); verifyResolvability(dataStreamName, indicesAdmin().preparePutMapping(dataStreamName).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(dataStreamName), false); + verifyResolvability(dataStreamName, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, dataStreamName), false); verifyResolvability( dataStreamName, indicesAdmin().prepareUpdateSettings(dataStreamName).setSettings(Settings.builder().put("index.number_of_replicas", 0)), @@ -643,7 +643,7 @@ public void testResolvabilityOfDataStreamsInAPIs() throws Exception { verifyResolvability(wildcardExpression, indicesAdmin().prepareGetFieldMappings(wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().preparePutMapping(wildcardExpression).setSource(""" {"_doc":{"properties": {"my_field":{"type":"keyword"}}}}""", XContentType.JSON), false); - verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(wildcardExpression), false); + verifyResolvability(wildcardExpression, indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, wildcardExpression), false); verifyResolvability(wildcardExpression, indicesAdmin().prepareGetSettings(wildcardExpression), false); verifyResolvability( wildcardExpression, @@ -1180,7 +1180,7 @@ public void testUpdateMappingViaDataStream() throws Exception { DataStreamTimestampFieldMapper.NAME, Map.of("enabled", true) ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); @@ -1195,7 +1195,7 @@ public void testUpdateMappingViaDataStream() throws Exception { .setSource("{\"properties\":{\"my_field\":{\"type\":\"keyword\"}}}", XContentType.JSON) .get(); // The mappings of all backing indices should be updated: - getMappingsResponse = indicesAdmin().prepareGetMappings("logs-foobar").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logs-foobar").get(); assertThat(getMappingsResponse.getMappings().size(), equalTo(2)); assertThat(getMappingsResponse.getMappings().get(backingIndex1).getSourceAsMap(), equalTo(expectedMapping)); assertThat(getMappingsResponse.getMappings().get(backingIndex2).getSourceAsMap(), equalTo(expectedMapping)); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index f090186480b76..8026ec641d040 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -9,10 +9,6 @@ package org.elasticsearch.datastreams; -import org.elasticsearch.action.admin.indices.rollover.LazyRolloverAction; -import org.elasticsearch.action.datastreams.autosharding.DataStreamAutoShardingService; -import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; -import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher; import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; @@ -27,12 +23,7 @@ public class DataStreamFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of( - DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE, // Added in 8.12 - LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER, // Added in 8.13 - DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE, - DataStreamGlobalRetention.GLOBAL_RETENTION // Added in 8.14 - ); + return Set.of(); } @Override diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java index cb7445705537a..7d5f4bbee32be 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java @@ -197,8 +197,7 @@ public Collection createComponents(PluginServices services) { settings, services.client(), services.clusterService(), - errorStoreInitialisationService.get(), - services.featureService() + errorStoreInitialisationService.get() ) ); dataLifecycleInitialisationService.set( diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java index 642fa4923e074..71575ee88aa7d 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisher.java @@ -19,8 +19,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.health.node.DataStreamLifecycleHealthInfo; import org.elasticsearch.health.node.DslErrorInfo; import org.elasticsearch.health.node.UpdateHealthInfoCacheAction; @@ -45,12 +43,10 @@ public class DataStreamLifecycleHealthInfoPublisher { Setting.Property.Dynamic, Setting.Property.NodeScope ); - public static final NodeFeature DSL_HEALTH_INFO_FEATURE = new NodeFeature("health.dsl.info", true); private final Client client; private final ClusterService clusterService; private final DataStreamLifecycleErrorStore errorStore; - private final FeatureService featureService; private volatile int signallingErrorRetryInterval; private volatile int maxNumberOfErrorsToPublish; @@ -58,13 +54,11 @@ public DataStreamLifecycleHealthInfoPublisher( Settings settings, Client client, ClusterService clusterService, - DataStreamLifecycleErrorStore errorStore, - FeatureService featureService + DataStreamLifecycleErrorStore errorStore ) { this.client = client; this.clusterService = clusterService; this.errorStore = errorStore; - this.featureService = featureService; this.signallingErrorRetryInterval = DATA_STREAM_SIGNALLING_ERROR_RETRY_INTERVAL_SETTING.get(settings); this.maxNumberOfErrorsToPublish = DATA_STREAM_LIFECYCLE_MAX_ERRORS_TO_PUBLISH_SETTING.get(settings); } @@ -89,9 +83,6 @@ private void updateNumberOfErrorsToPublish(int newValue) { * {@link org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService#DATA_STREAM_SIGNALLING_ERROR_RETRY_INTERVAL_SETTING} */ public void publishDslErrorEntries(ActionListener actionListener) { - if (featureService.clusterHasFeature(clusterService.state(), DSL_HEALTH_INFO_FEATURE) == false) { - return; - } // fetching the entries that persist in the error store for more than the signalling retry interval // note that we're reporting this view into the error store on every publishing iteration List errorEntriesToSignal = errorStore.getErrorsInfo( diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index ac7dabd868a3f..0bb990e544892 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -67,9 +67,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; import org.elasticsearch.core.Tuple; -import org.elasticsearch.datastreams.DataStreamFeatures; import org.elasticsearch.datastreams.lifecycle.health.DataStreamLifecycleHealthInfoPublisher; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; @@ -183,13 +181,7 @@ public void setupServices() { () -> now, errorStore, allocationService, - new DataStreamLifecycleHealthInfoPublisher( - Settings.EMPTY, - client, - clusterService, - errorStore, - new FeatureService(List.of(new DataStreamFeatures())) - ), + new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, client, clusterService, errorStore), globalRetentionSettings ); clientDelegate = null; @@ -1465,13 +1457,7 @@ public void testTrackingTimeStats() { () -> now.getAndAdd(delta), errorStore, mock(AllocationService.class), - new DataStreamLifecycleHealthInfoPublisher( - Settings.EMPTY, - getTransportRequestsRecordingClient(), - clusterService, - errorStore, - new FeatureService(List.of(new DataStreamFeatures())) - ), + new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, getTransportRequestsRecordingClient(), clusterService, errorStore), globalRetentionSettings ); assertThat(service.getLastRunDuration(), is(nullValue())); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java index cff6127e0729e..f8a2ac3c61029 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/health/DataStreamLifecycleHealthInfoPublisherTests.java @@ -24,10 +24,8 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.datastreams.DataStreamFeatures; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleErrorStore; import org.elasticsearch.datastreams.lifecycle.DataStreamLifecycleService; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.health.node.DataStreamLifecycleHealthInfo; import org.elasticsearch.health.node.DslErrorInfo; import org.elasticsearch.health.node.UpdateHealthInfoCacheAction; @@ -40,7 +38,6 @@ import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; @@ -83,13 +80,7 @@ public void setupServices() { final Client client = getTransportRequestsRecordingClient(); errorStore = new DataStreamLifecycleErrorStore(() -> now); - dslHealthInfoPublisher = new DataStreamLifecycleHealthInfoPublisher( - Settings.EMPTY, - client, - clusterService, - errorStore, - new FeatureService(List.of(new DataStreamFeatures())) - ); + dslHealthInfoPublisher = new DataStreamLifecycleHealthInfoPublisher(Settings.EMPTY, client, clusterService, errorStore); } @After @@ -105,16 +96,6 @@ public void testPublishDslErrorEntries() { } errorStore.recordError("testIndex", new IllegalStateException("bad state")); ClusterState stateWithHealthNode = ClusterStateCreationUtils.state(node1, node1, node1, allNodes); - stateWithHealthNode = ClusterState.builder(stateWithHealthNode) - .nodeFeatures( - Map.of( - node1.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()), - node2.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()) - ) - ) - .build(); ClusterServiceUtils.setState(clusterService, stateWithHealthNode); dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() { @Override @@ -143,16 +124,6 @@ public void testPublishDslErrorEntriesNoHealthNode() { errorStore.recordError("testIndex", new IllegalStateException("bad state")); ClusterState stateNoHealthNode = ClusterStateCreationUtils.state(node1, node1, null, allNodes); - stateNoHealthNode = ClusterState.builder(stateNoHealthNode) - .nodeFeatures( - Map.of( - node1.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()), - node2.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()) - ) - ) - .build(); ClusterServiceUtils.setState(clusterService, stateNoHealthNode); dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() { @Override @@ -170,16 +141,6 @@ public void onFailure(Exception e) { public void testPublishDslErrorEntriesEmptyErrorStore() { // publishes the empty error store (this is the "back to healthy" state where all errors have been fixed) ClusterState state = ClusterStateCreationUtils.state(node1, node1, node1, allNodes); - state = ClusterState.builder(state) - .nodeFeatures( - Map.of( - node1.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()), - node2.getId(), - Set.of(DataStreamLifecycleHealthInfoPublisher.DSL_HEALTH_INFO_FEATURE.id()) - ) - ) - .build(); ClusterServiceUtils.setState(clusterService, state); dslHealthInfoPublisher.publishDslErrorEntries(new ActionListener<>() { @Override diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 9ea3bfefabdf8..884adb5458102 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -300,9 +300,6 @@ index without timestamp with pipeline: --- dynamic templates: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -450,9 +447,6 @@ dynamic templates: --- dynamic templates - conflicting aliases: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -549,9 +543,6 @@ dynamic templates - conflicting aliases: --- dynamic templates - conflicting aliases with top-level field: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [otel] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -632,9 +623,6 @@ dynamic templates - conflicting aliases with top-level field: --- dynamic templates with nesting: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -810,10 +798,6 @@ dynamic templates with nesting: --- dynamic templates with incremental indexing: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -1038,9 +1022,6 @@ dynamic templates with incremental indexing: --- subobject in passthrough object auto flatten: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-passthrough-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-passthrough-template] will take precedence during new index creation" @@ -1108,9 +1089,6 @@ enable subobjects in passthrough object: --- passthrough objects with duplicate priority: - - requires: - cluster_features: ["mapper.pass_through_priority"] - reason: support for priority in passthrough objects - do: catch: /has a conflicting param/ indices.put_index_template: @@ -1135,9 +1113,6 @@ passthrough objects with duplicate priority: --- dimensions with ignore_malformed and ignore_above: - - requires: - cluster_features: ["mapper.keyword_dimension_ignore_above"] - reason: support for ignore_above on keyword dimensions - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -1229,9 +1204,6 @@ dimensions with ignore_malformed and ignore_above: --- non string dimension fields: - - requires: - cluster_features: ["mapper.pass_through_priority", "routing.boolean_routing_path", "mapper.boolean_dimension"] - reason: support for priority in passthrough objects - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" @@ -1339,10 +1311,6 @@ non string dimension fields: --- multi value dimensions: - - requires: - cluster_features: ["routing.multi_value_routing_path"] - reason: support for multi-value dimensions - - do: allowed_warnings: - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java index aa48c73cf1d73..08efe87e6fde5 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/DatabaseConfiguration.java @@ -160,11 +160,6 @@ public void writeTo(StreamOutput out) throws IOException { if (provider instanceof Maxmind maxmind) { out.writeString(maxmind.accountId); } else { - /* - * The existence of a non-Maxmind providers is gated on the feature get_database_configuration_action.multi_node, and - * get_database_configuration_action.multi_node is only available on or after - * TransportVersions.INGEST_GEO_DATABASE_PROVIDERS. - */ assert false : "non-maxmind DatabaseConfiguration.Provider [" + provider.getWriteableName() + "]"; } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java index c83c40e56b749..a1faaf1bb0196 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportGetDatabaseConfigurationAction.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.geoip.DatabaseNodeService; import org.elasticsearch.ingest.geoip.GeoIpTaskState; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; @@ -41,8 +40,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.elasticsearch.ingest.IngestGeoIpFeatures.GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE; - public class TransportGetDatabaseConfigurationAction extends TransportNodesAction< GetDatabaseConfigurationAction.Request, GetDatabaseConfigurationAction.Response, @@ -50,7 +47,6 @@ public class TransportGetDatabaseConfigurationAction extends TransportNodesActio GetDatabaseConfigurationAction.NodeResponse, List> { - private final FeatureService featureService; private final DatabaseNodeService databaseNodeService; @Inject @@ -59,7 +55,6 @@ public TransportGetDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - FeatureService featureService, DatabaseNodeService databaseNodeService ) { super( @@ -70,39 +65,9 @@ public TransportGetDatabaseConfigurationAction( GetDatabaseConfigurationAction.NodeRequest::new, threadPool.executor(ThreadPool.Names.MANAGEMENT) ); - this.featureService = featureService; this.databaseNodeService = databaseNodeService; } - @Override - protected void doExecute( - Task task, - GetDatabaseConfigurationAction.Request request, - ActionListener listener - ) { - if (featureService.clusterHasFeature(clusterService.state(), GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE) == false) { - /* - * TransportGetDatabaseConfigurationAction used to be a TransportMasterNodeAction, and not all nodes in the cluster have been - * updated. So we don't want to send node requests to the other nodes because they will blow up. Instead, we just return - * the information that we used to return from the master node (it doesn't make any difference that this might not be the master - * node, because we're only reading the cluster state). Because older nodes only know about the Maxmind provider type, we filter - * out all others here to avoid causing problems on those nodes. - */ - newResponseAsync( - task, - request, - createActionContext(task, request).stream() - .filter(database -> database.database().provider() instanceof DatabaseConfiguration.Maxmind) - .toList(), - List.of(), - List.of(), - listener - ); - } else { - super.doExecute(task, request, listener); - } - } - protected List createActionContext(Task task, GetDatabaseConfigurationAction.Request request) { final Set ids; if (request.getDatabaseIds().length == 0) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java index dfb8fa78089d2..e68bb9d82e91b 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/direct/TransportPutDatabaseConfigurationAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.core.Tuple; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.ingest.geoip.IngestGeoIpMetadata; import org.elasticsearch.ingest.geoip.direct.PutDatabaseConfigurationAction.Request; import org.elasticsearch.injection.guice.Inject; @@ -42,8 +41,6 @@ import java.util.Map; import java.util.Optional; -import static org.elasticsearch.ingest.IngestGeoIpFeatures.PUT_DATABASE_CONFIGURATION_ACTION_IPINFO; - public class TransportPutDatabaseConfigurationAction extends TransportMasterNodeAction { private static final Logger logger = LogManager.getLogger(TransportPutDatabaseConfigurationAction.class); @@ -61,7 +58,6 @@ public void taskSucceeded(UpdateDatabaseConfigurationTask task, Void unused) { } }; - private final FeatureService featureService; private final MasterServiceTaskQueue updateDatabaseConfigurationTaskQueue; @Inject @@ -70,8 +66,7 @@ public TransportPutDatabaseConfigurationAction( ClusterService clusterService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, - FeatureService featureService + IndexNameExpressionResolver indexNameExpressionResolver ) { super( PutDatabaseConfigurationAction.NAME, @@ -84,7 +79,6 @@ public TransportPutDatabaseConfigurationAction( AcknowledgedResponse::readFrom, EsExecutors.DIRECT_EXECUTOR_SERVICE ); - this.featureService = featureService; this.updateDatabaseConfigurationTaskQueue = clusterService.createTaskQueue( "update-geoip-database-configuration-state-update", Priority.NORMAL, @@ -96,18 +90,6 @@ public TransportPutDatabaseConfigurationAction( protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { final String id = request.getDatabase().id(); - // if this is an ipinfo configuration, then make sure the whole cluster supports that feature - if (request.getDatabase().provider() instanceof DatabaseConfiguration.Ipinfo - && featureService.clusterHasFeature(clusterService.state(), PUT_DATABASE_CONFIGURATION_ACTION_IPINFO) == false) { - listener.onFailure( - new IllegalArgumentException( - "Unable to use ipinfo database configurations in mixed-clusters with nodes that do not support feature " - + PUT_DATABASE_CONFIGURATION_ACTION_IPINFO.id() - ) - ); - return; - } - updateDatabaseConfigurationTaskQueue.submitTask( Strings.format("update-geoip-database-configuration-[%s]", id), new UpdateDatabaseConfigurationTask(listener, request.getDatabase()), diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml index a1104505bc240..007c82db4c923 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/40_geoip_databases.yml @@ -1,9 +1,3 @@ ---- -setup: - - requires: - cluster_features: ["geoip.downloader.database.configuration", "get_database_configuration_action.multi_node"] - reason: "geoip downloader database configuration APIs added in 8.15, and updated in 8.16 to return more results" - --- teardown: - do: diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml index fd73c715a5ac5..0947984769529 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/50_ip_lookup_processor.yml @@ -1,9 +1,3 @@ -setup: - - requires: - cluster_features: - - "put_database_configuration_action.ipinfo" - reason: "ipinfo support added in 8.16" - --- "Test ip_location processor with defaults": - do: diff --git a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml index e2e9a1fdb5e28..47f09392df60e 100644 --- a/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml +++ b/modules/ingest-geoip/src/yamlRestTest/resources/rest-api-spec/test/ingest_geoip/60_ip_location_databases.yml @@ -1,10 +1,3 @@ ---- -setup: - - requires: - cluster_features: - - "put_database_configuration_action.ipinfo" - reason: "ip location downloader database configuration APIs added in 8.16 to support more types" - --- teardown: - do: diff --git a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java index 553e4696af316..a9ab0c02612f6 100644 --- a/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java +++ b/modules/kibana/src/internalClusterTest/java/org/elasticsearch/kibana/KibanaThreadPoolIT.java @@ -16,6 +16,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; @@ -23,7 +25,6 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPoolStats; @@ -49,10 +50,6 @@ * threads that wait on a phaser. This lets us verify that operations on system indices * are being directed to other thread pools.

*/ -@TestLogging( - reason = "investigate", - value = "org.elasticsearch.kibana.KibanaThreadPoolIT:DEBUG,org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor:TRACE" -) public class KibanaThreadPoolIT extends ESIntegTestCase { private static final Logger logger = LogManager.getLogger(KibanaThreadPoolIT.class); @@ -68,6 +65,8 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .put("thread_pool.write.queue_size", 1) .put("thread_pool.get.size", 1) .put("thread_pool.get.queue_size", 1) + // a rejected GET may retry on an INITIALIZING shard (the target of a relocation) and unexpectedly succeed, so block rebalancing + .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE) .build(); } @@ -112,7 +111,12 @@ public void testKibanaThreadPoolByPassesBlockedThreadPools() throws Exception { } public void testBlockedThreadPoolsRejectUserRequests() throws Exception { - assertAcked(client().admin().indices().prepareCreate(USER_INDEX)); + assertAcked( + client().admin() + .indices() + .prepareCreate(USER_INDEX) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) // avoid retrying rejected actions + ); runWithBlockedThreadPools(this::assertThreadPoolsBlocked); diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml index 25088f51e2b59..1434450b65a6a 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml @@ -221,9 +221,6 @@ setup: - close_to: {hits.hits.2._score: {value: 186.34454, error: 0.01}} --- "Test hamming distance fails on float": - - requires: - cluster_features: ["script.hamming"] - reason: "support for hamming distance added in 8.15" - do: headers: Content-Type: application/json diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml index cdd65ca0eb296..05a10ffdbccdb 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/146_dense_vector_bit_basic.yml @@ -1,7 +1,5 @@ setup: - requires: - cluster_features: ["mapper.vectors.bit_vectors"] - reason: "support for bit vectors added in 8.15" test_runner_features: headers - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml index 373f048e7be78..a6c111be681f9 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml @@ -1,7 +1,5 @@ setup: - requires: - cluster_features: ["script.hamming"] - reason: "support for hamming distance added in 8.15" test_runner_features: headers - do: diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml index f82b844f01588..3a869640993f4 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/190_term_statistics_script_score.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: ["script.term_stats"] - reason: "support for term stats has been added in 8.16" - - do: indices.create: index: test-index diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml index de4d6530f4a92..3a9c71e3c2bab 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/191_term_statistics_function_score.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: ["script.term_stats"] - reason: "support for term stats has been added in 8.16" - - do: indices.create: index: test-index diff --git a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml index 968e93cf9fc55..175abe183106b 100644 --- a/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml +++ b/modules/repository-azure/src/yamlRestTest/resources/rest-api-spec/test/repository_azure/20_repository.yml @@ -251,11 +251,6 @@ setup: --- "Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - do: cluster.stats: {} diff --git a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml index e8c34a4b6a20b..d2370919297a3 100644 --- a/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml +++ b/modules/repository-gcs/src/yamlRestTest/resources/rest-api-spec/test/repository_gcs/20_repository.yml @@ -234,11 +234,6 @@ setup: --- "Usage stats": - - requires: - cluster_features: - - repositories.supports_usage_stats - reason: requires this feature - - do: cluster.stats: {} diff --git a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java index 93915e8491d5b..3d7c8dd150610 100644 --- a/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java +++ b/modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3MinioBasicCredentialsRestIT.java @@ -19,13 +19,15 @@ import org.junit.rules.RuleChain; import org.junit.rules.TestRule; +import java.util.Locale; + @ThreadLeakFilters(filters = { TestContainersThreadFilter.class }) @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482 public class RepositoryS3MinioBasicCredentialsRestIT extends AbstractRepositoryS3RestTestCase { - private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT"); + private static final String PREFIX = getIdentifierPrefix("RepositoryS3MinioBasicCredentialsRestIT").toLowerCase(Locale.ROOT); private static final String BUCKET = PREFIX + "bucket"; - private static final String BASE_PATH = PREFIX + "base_path"; + private static final String BASE_PATH = PREFIX + "base-path"; private static final String ACCESS_KEY = PREFIX + "access-key"; private static final String SECRET_KEY = PREFIX + "secret-key"; private static final String CLIENT = "minio_client"; diff --git a/muted-tests.yml b/muted-tests.yml index 2f652f87ab283..9766d3ed35f18 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -52,9 +52,6 @@ tests: - class: org.elasticsearch.xpack.transform.integration.TransformIT method: testStopWaitForCheckpoint issue: https://github.com/elastic/elasticsearch/issues/106113 -- class: org.elasticsearch.kibana.KibanaThreadPoolIT - method: testBlockedThreadPoolsRejectUserRequests - issue: https://github.com/elastic/elasticsearch/issues/113939 - class: org.elasticsearch.xpack.inference.TextEmbeddingCrudIT method: testPutE5Small_withPlatformAgnosticVariant issue: https://github.com/elastic/elasticsearch/issues/113983 @@ -227,50 +224,45 @@ tests: - class: org.elasticsearch.search.profile.dfs.DfsProfilerIT method: testProfileDfs issue: https://github.com/elastic/elasticsearch/issues/119711 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchFunctionFilterPushdownWithStringValues {default} - issue: https://github.com/elastic/elasticsearch/issues/119720 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchFunctionPushdownWithCasting {default} - issue: https://github.com/elastic/elasticsearch/issues/119722 -- class: org.elasticsearch.xpack.esql.optimizer.LocalPhysicalPlanOptimizerTests - method: testSingleMatchOperatorFilterPushdownWithStringValues {default} - issue: https://github.com/elastic/elasticsearch/issues/119721 -- class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterIT - method: testBulkOperations {p0=false} - issue: https://github.com/elastic/elasticsearch/issues/119901 - class: org.elasticsearch.xpack.inference.InferenceCrudIT method: testGetServicesWithCompletionTaskType issue: https://github.com/elastic/elasticsearch/issues/119959 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119978 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119979 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 8.18.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119550 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 8.18.0]} - issue: https://github.com/elastic/elasticsearch/issues/119980 -- class: org.elasticsearch.index.codec.vectors.es816.ES816HnswBinaryQuantizedVectorsFormatTests - method: testRandomExceptions - issue: https://github.com/elastic/elasticsearch/issues/119981 - class: org.elasticsearch.multi_cluster.MultiClusterYamlTestSuiteIT issue: https://github.com/elastic/elasticsearch/issues/119983 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testMountSearchableSnapshot {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119989 -- class: org.elasticsearch.lucene.RollingUpgradeSearchableSnapshotIndexCompatibilityIT - method: testSearchableSnapshotUpgrade {p0=[9.0.0, 9.0.0, 9.0.0]} - issue: https://github.com/elastic/elasticsearch/issues/119990 - class: org.elasticsearch.xpack.test.rest.XPackRestIT method: test {p0=transform/transforms_unattended/Test unattended put and start} issue: https://github.com/elastic/elasticsearch/issues/120019 - class: org.elasticsearch.index.mapper.IntervalThrottlerTests method: testThrottling issue: https://github.com/elastic/elasticsearch/issues/120023 +- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT + method: testCheckThrows {pathPrefix=denied actionName=sslSessionImpl_getSessionContext} + issue: https://github.com/elastic/elasticsearch/issues/120053 +- class: org.elasticsearch.entitlement.qa.EntitlementsDeniedIT + method: testCheckThrows {pathPrefix=denied_nonmodular actionName=sslSessionImpl_getSessionContext} + issue: https://github.com/elastic/elasticsearch/issues/120054 +- class: org.elasticsearch.xpack.ilm.actions.SearchableSnapshotActionIT + method: testUpdatePolicyToAddPhasesYieldsInvalidActionsToBeSkipped + issue: https://github.com/elastic/elasticsearch/issues/118406 +- class: org.elasticsearch.xpack.ml.integration.DatafeedJobsIT + issue: https://github.com/elastic/elasticsearch/issues/120088 +- class: org.elasticsearch.xpack.searchablesnapshots.minio.MinioSearchableSnapshotsIT + issue: https://github.com/elastic/elasticsearch/issues/120101 +- class: org.elasticsearch.repositories.s3.S3RepositoryThirdPartyTests + issue: https://github.com/elastic/elasticsearch/issues/120115 +- class: org.elasticsearch.repositories.s3.RepositoryS3MinioBasicCredentialsRestIT + issue: https://github.com/elastic/elasticsearch/issues/120117 +- class: org.elasticsearch.repositories.blobstore.testkit.analyze.MinioRepositoryAnalysisRestIT + issue: https://github.com/elastic/elasticsearch/issues/118548 +- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT + method: testConfiguredReservedRolesAfterClosingAndOpeningIndex + issue: https://github.com/elastic/elasticsearch/issues/120127 +- class: org.elasticsearch.oldrepos.OldRepositoryAccessIT + method: testOldRepoAccess + issue: https://github.com/elastic/elasticsearch/issues/120148 +- class: org.elasticsearch.oldrepos.OldRepositoryAccessIT + method: testOldSourceOnlyRepoAccess + issue: https://github.com/elastic/elasticsearch/issues/120080 # Examples: # diff --git a/plugins/mapper-annotated-text/src/main/java/module-info.java b/plugins/mapper-annotated-text/src/main/java/module-info.java index 13f2bd66418be..58aca0d2857fe 100644 --- a/plugins/mapper-annotated-text/src/main/java/module-info.java +++ b/plugins/mapper-annotated-text/src/main/java/module-info.java @@ -15,6 +15,4 @@ requires org.apache.lucene.highlighter; // exports nothing - - provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.index.mapper.annotatedtext.Features; } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 33b5db1c4662d..4b2006430b89e 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -64,8 +63,6 @@ **/ public class AnnotatedTextFieldMapper extends FieldMapper { - public static final NodeFeature SYNTHETIC_SOURCE_SUPPORT = new NodeFeature("mapper.annotated_text.synthetic_source", true); - public static final String CONTENT_TYPE = "annotated_text"; private static Builder builder(FieldMapper in) { diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java deleted file mode 100644 index 51a2d2bbe1d40..0000000000000 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/Features.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index.mapper.annotatedtext; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -/** - * Provides features for annotated text mapper. - */ -public class Features implements FeatureSpecification { - @Override - public Set getFeatures() { - return Set.of( - AnnotatedTextFieldMapper.SYNTHETIC_SOURCE_SUPPORT // Added in 8.15 - ); - } -} diff --git a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification deleted file mode 100644 index 1fc11da18fc3c..0000000000000 --- a/plugins/mapper-annotated-text/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ /dev/null @@ -1,10 +0,0 @@ -# - # Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - # or more contributor license agreements. Licensed under the "Elastic License - # 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - # Public License v 1"; you may not use this file except in compliance with, at - # your election, the "Elastic License 2.0", the "GNU Affero General Public - # License v3.0 only", or the "Server Side Public License, v 1". -# - -org.elasticsearch.index.mapper.annotatedtext.Features diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index c2251910c3122..435849821691e 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -91,7 +91,7 @@ private void assertSizeMappingEnabled(String index, boolean enabled) throws IOEx "Expected size field mapping to be " + (enabled ? "enabled" : "disabled") + " for %s", index ); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); Map mappingSource = getMappingsResponse.getMappings().get(index).getSourceAsMap(); assertThat(errMsg, mappingSource, hasKey("_size")); String sizeAsString = mappingSource.get("_size").toString(); diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java index d9adec47ff483..30367bf55d8cc 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java @@ -11,7 +11,6 @@ import com.carrotsearch.randomizedtesting.annotations.Name; -import org.elasticsearch.Build; import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest; import org.elasticsearch.client.Request; import org.elasticsearch.cluster.metadata.DesiredNode; @@ -84,7 +83,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve randomDoubleProcessorCount(), ByteSizeValue.ofGb(randomIntBetween(10, 24)), ByteSizeValue.ofGb(randomIntBetween(128, 256)), - clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version() + null ) ) .toList(); @@ -96,7 +95,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)), ByteSizeValue.ofGb(randomIntBetween(10, 24)), ByteSizeValue.ofGb(randomIntBetween(128, 256)), - clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version() + null ); }).toList(); } diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java index ce514c5f1b1e7..c48ae9ba1843b 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/RestActionCancellationIT.java @@ -12,12 +12,15 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.template.get.GetComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.post.SimulateTemplateAction; import org.elasticsearch.action.support.CancellableActionTestPlugin; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; @@ -81,6 +84,25 @@ public void testGetComposableTemplateCancellation() { runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_index_template"), GetComposableIndexTemplateAction.NAME); } + public void testSimulateTemplateCancellation() { + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate/random_index_template"), + SimulateTemplateAction.NAME + ); + } + + public void testSimulateIndexTemplateCancellation() { + createIndex("test"); + runRestActionCancellationTest( + new Request(HttpPost.METHOD_NAME, "/_index_template/_simulate_index/test"), + SimulateIndexTemplateAction.NAME + ); + } + + public void testClusterGetSettingsCancellation() { + runRestActionCancellationTest(new Request(HttpGet.METHOD_NAME, "/_cluster/settings"), ClusterGetSettingsAction.NAME); + } + private void runRestActionCancellationTest(Request request, String actionName) { final var node = usually() ? internalCluster().getRandomNodeName() : internalCluster().startCoordinatingOnlyNode(Settings.EMPTY); diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index d4843fb152888..4a5ceeb66f661 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -222,10 +222,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.mapping.validation.templates"] - reason: "ingest simulate index mapping validation added in 8.16" - - do: headers: Content-Type: application/json @@ -313,10 +309,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.component.template.substitutions"] - reason: "ingest simulate component template substitutions added in 8.16" - - do: headers: Content-Type: application/json @@ -494,10 +486,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.component.template.substitutions"] - reason: "ingest simulate component template substitutions added in 8.16" - - do: headers: Content-Type: application/json @@ -617,10 +605,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.component.template.substitutions"] - reason: "ingest simulate component template substitutions added in 8.16" - - do: headers: Content-Type: application/json @@ -816,10 +800,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.index.template.substitutions"] - reason: "ingest simulate index template substitutions added in 8.16" - - do: headers: Content-Type: application/json @@ -1010,10 +990,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.index.template.substitutions"] - reason: "ingest simulate component template substitutions added in 8.16" - - do: headers: Content-Type: application/json @@ -1227,10 +1203,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.mapping.addition"] - reason: "ingest simulate mapping addition added in 8.17" - - do: headers: Content-Type: application/json @@ -1463,10 +1435,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.mapping.addition"] - reason: "ingest simulate mapping addition added in 8.17" - - do: indices.put_template: name: my-legacy-template @@ -1584,10 +1552,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.support.non.template.mapping"] - reason: "ingest simulate support for indices with mappings that didn't come from templates added in 8.17" - # A global match-everything legacy template is added to the cluster sometimes (rarely). We have to get rid of this template if it exists # because this test is making sure we get correct behavior when an index matches *no* template: - do: diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 1da8e906582b1..e4b46b98cedda 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -85,4 +85,6 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport") task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport") task.skipTest("search.highlight/30_max_analyzed_offset/Plain highlighter with max_analyzed_offset < 0 should FAIL", "semantics of test has changed") + task.skipTest("indices.create/10_basic/Create lookup index", "default auto_expand_replicas was removed") + task.skipTest("indices.create/10_basic/Create lookup index with one shard", "default auto_expand_replicas was removed") }) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json index 5862804257c67..5004ab8de697d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json @@ -26,7 +26,7 @@ }, "master_timeout":{ "type":"time", - "description":"Explicit operation timeout for connection to master node" + "description":"Timeout for waiting for new cluster state in case it is blocked" }, "timeout":{ "type":"time", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json deleted file mode 100644 index 2327519ff2816..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.unfreeze.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "indices.unfreeze":{ - "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/unfreeze-index-api.html", - "description":"Unfreezes an index. When a frozen index is unfrozen, the index goes through the normal recovery process and becomes writeable again." - }, - "stability":"stable", - "visibility":"public", - "headers":{ - "accept": [ "application/json"] - }, - "url":{ - "paths":[ - { - "path":"/{index}/_unfreeze", - "methods":[ - "POST" - ], - "parts":{ - "index":{ - "type":"string", - "description":"The name of the index to unfreeze" - } - }, - "deprecated":{ - "version":"7.14.0", - "description":"Frozen indices are deprecated because they provide no benefit given improvements in heap memory utilization. They will be removed in a future release." - } - } - ] - }, - "params":{ - "timeout":{ - "type":"time", - "description":"Explicit operation timeout" - }, - "master_timeout":{ - "type":"time", - "description":"Specify timeout for connection to master" - }, - "ignore_unavailable":{ - "type":"boolean", - "description":"Whether specified concrete indices should be ignored when unavailable (missing or closed)" - }, - "allow_no_indices":{ - "type":"boolean", - "description":"Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes `_all` string or when no indices have been specified)" - }, - "expand_wildcards":{ - "type":"enum", - "options":[ - "open", - "closed", - "hidden", - "none", - "all" - ], - "default":"closed", - "description":"Whether to expand wildcard expression to concrete indices that are open, closed or both." - }, - "wait_for_active_shards":{ - "type":"string", - "description":"Sets the number of active shards to wait for before the operation returns." - } - } - } -} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml index 9da6d2c5f086e..ce3f7f0198399 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/field_caps/70_index_mode.yml @@ -1,9 +1,5 @@ --- setup: - - requires: - cluster_features: "mapper.query_index_mode" - reason: "require index_mode" - - do: indices.create: index: test_metrics diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index 13f6ca58ea295..a0061272a2c23 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -1014,10 +1014,6 @@ flattened field: --- flattened field with ignore_above: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -1070,10 +1066,6 @@ flattened field with ignore_above: --- flattened field with ignore_above and arrays: - - requires: - cluster_features: ["mapper.flattened.ignore_above_with_arrays_support"] - reason: requires support of ignore_above synthetic source with arrays - - do: indices.create: index: test @@ -1127,10 +1119,6 @@ flattened field with ignore_above and arrays: --- completion: - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml index 414c24cfffd7d..7b8f785a2cb93 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/index/92_metrics_auto_subobjects.yml @@ -2,7 +2,6 @@ "Metrics object indexing": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] reason: requires supporting subobjects auto setting - do: @@ -69,7 +68,6 @@ "Root with metrics": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] reason: requires supporting subobjects auto setting - do: @@ -131,7 +129,6 @@ "Metrics object indexing with synthetic source": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] reason: added in 8.4.0 - do: @@ -201,7 +198,6 @@ "Root without subobjects with synthetic source": - requires: test_runner_features: [ "allowed_warnings", "allowed_warnings_regex" ] - cluster_features: ["mapper.subobjects_auto"] reason: added in 8.4.0 - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml index d0e1759073e1b..8645c91a51ad3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml @@ -171,7 +171,6 @@ index: test_lookup - match: { test_lookup.settings.index.number_of_shards: "1"} - - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} --- "Create lookup index with one shard": @@ -196,7 +195,6 @@ index: test_lookup - match: { test_lookup.settings.index.number_of_shards: "1"} - - match: { test_lookup.settings.index.auto_expand_replicas: "0-all"} --- "Create lookup index with two shards": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 5003f6df79a14..72dddcf8052cc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -6,7 +6,7 @@ setup: --- object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -56,10 +56,6 @@ object with unmapped fields: --- unmapped arrays: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -103,7 +99,7 @@ unmapped arrays: --- nested object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -154,7 +150,7 @@ nested object with unmapped fields: --- empty object with unmapped fields: - requires: - cluster_features: ["mapper.track_ignored_source", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -314,10 +310,6 @@ disabled object contains array: --- disabled subobject: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -353,10 +345,6 @@ disabled subobject: --- disabled subobject with array: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -393,10 +381,6 @@ disabled subobject with array: --- mixed disabled and enabled objects: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -488,10 +472,6 @@ object with dynamic override: --- subobject with dynamic override: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -537,10 +517,6 @@ subobject with dynamic override: --- object array in object with dynamic override: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -592,10 +568,6 @@ object array in object with dynamic override: --- value array in object with dynamic override: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -636,10 +608,6 @@ value array in object with dynamic override: --- nested object: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -682,10 +650,6 @@ nested object: --- nested object next to regular: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -729,10 +693,6 @@ nested object next to regular: --- nested object with disabled: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -818,10 +778,6 @@ nested object with disabled: --- doubly nested object: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -915,7 +871,7 @@ doubly nested object: --- subobjects auto: - requires: - cluster_features: ["mapper.subobjects_auto", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source and supporting subobjects auto setting - do: @@ -1003,10 +959,6 @@ subobjects auto: --- synthetic_source with copy_to: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1140,10 +1092,6 @@ synthetic_source with copy_to: --- synthetic_source with disabled doc_values: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires disabled doc_values support in synthetic source - - do: indices.create: index: test @@ -1224,10 +1172,6 @@ synthetic_source with disabled doc_values: --- fallback synthetic_source for text field: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires disabled doc_values support in synthetic source - - do: indices.create: index: test @@ -1259,10 +1203,6 @@ fallback synthetic_source for text field: --- synthetic_source with copy_to and ignored values: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1328,10 +1268,6 @@ synthetic_source with copy_to and ignored values: --- synthetic_source with copy_to field having values in source: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1392,10 +1328,6 @@ synthetic_source with copy_to field having values in source: --- synthetic_source with ignored source field using copy_to: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1457,10 +1389,6 @@ synthetic_source with ignored source field using copy_to: --- synthetic_source with copy_to field from dynamic template having values in source: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1555,7 +1483,6 @@ synthetic_source with copy_to field from dynamic template having values in sourc --- synthetic_source with copy_to and invalid values for copy: - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_fix"] reason: requires copy_to support in synthetic source test_runner_features: "contains" @@ -1592,10 +1519,6 @@ synthetic_source with copy_to and invalid values for copy: --- synthetic_source with copy_to pointing inside object: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1697,10 +1620,6 @@ synthetic_source with copy_to pointing inside object: --- synthetic_source with copy_to pointing to ambiguous field: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1745,10 +1664,6 @@ synthetic_source with copy_to pointing to ambiguous field: --- synthetic_source with copy_to pointing to ambiguous field and subobjects false: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1794,10 +1709,6 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects false: --- synthetic_source with copy_to pointing to ambiguous field and subobjects auto: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test @@ -1845,7 +1756,6 @@ synthetic_source with copy_to pointing to ambiguous field and subobjects auto: synthetic_source with copy_to pointing at dynamic field: - requires: test_runner_features: contains - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] reason: requires copy_to support in synthetic source - do: @@ -1931,10 +1841,6 @@ synthetic_source with copy_to pointing at dynamic field: --- synthetic_source with copy_to pointing inside dynamic object: - - requires: - cluster_features: ["mapper.source.synthetic_source_copy_to_inside_objects_fix"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml index 095665e9337b1..803b8a7d0062f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/21_synthetic_source_stored.yml @@ -6,7 +6,7 @@ setup: --- object param - store complex object: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -71,7 +71,7 @@ object param - store complex object: --- object param - object array: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -136,7 +136,7 @@ object param - object array: --- object param - object array within array: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -180,7 +180,7 @@ object param - object array within array: --- object param - no object array: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -223,7 +223,7 @@ object param - no object array: --- object param - field ordering in object array: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -273,7 +273,7 @@ object param - field ordering in object array: --- object param - nested object array next to other fields: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -380,7 +380,7 @@ object param - nested object with stored array: --- index param - nested array within array: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires tracking ignored source - do: @@ -428,7 +428,7 @@ index param - nested array within array: # 112156 stored field under object with store_array_source: - requires: - cluster_features: ["mapper.source.synthetic_source_stored_fields_advance_fix", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires bug fix to be implemented - do: @@ -477,10 +477,6 @@ stored field under object with store_array_source: --- field param - keep root array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -535,10 +531,6 @@ field param - keep root array: --- field param - keep nested array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -605,7 +597,6 @@ field param - keep nested array: field param - keep root singleton fields: - requires: test_runner_features: close_to - cluster_features: ["mapper.synthetic_source_keep"] reason: requires keeping singleton source - do: @@ -695,7 +686,6 @@ field param - keep root singleton fields: field param - keep nested singleton fields: - requires: test_runner_features: close_to - cluster_features: ["mapper.synthetic_source_keep"] reason: requires keeping singleton source - do: @@ -776,10 +766,6 @@ field param - keep nested singleton fields: --- field param - nested array within array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires tracking ignored source - - do: indices.create: index: test @@ -821,7 +807,7 @@ field param - nested array within array: --- index param - root arrays: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires keeping array source - do: @@ -900,10 +886,6 @@ index param - root arrays: --- index param - dynamic root arrays: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -952,10 +934,6 @@ index param - dynamic root arrays: --- index param - object array within array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -1001,10 +979,6 @@ index param - object array within array: --- index param - no object array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -1045,10 +1019,6 @@ index param - no object array: --- index param - field ordering: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -1095,10 +1065,6 @@ index param - field ordering: --- index param - nested arrays: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -1162,10 +1128,6 @@ index param - nested arrays: --- index param - nested object with stored array: - - requires: - cluster_features: ["mapper.synthetic_source_keep"] - reason: requires keeping array source - - do: indices.create: index: test @@ -1214,7 +1176,7 @@ index param - nested object with stored array: --- index param - flattened fields: - requires: - cluster_features: ["mapper.synthetic_source_keep", "mapper.bwc_workaround_9_0"] + cluster_features: ["mapper.bwc_workaround_9_0"] reason: requires keeping array source - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml index 3d82539944a97..89816be5ca8e7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.put_index_template/15_composition.yml @@ -453,8 +453,6 @@ --- "Composable index templates that include subobjects: auto at root": - requires: - cluster_features: ["mapper.subobjects_auto"] - reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" test_runner_features: "allowed_warnings" - do: @@ -504,8 +502,6 @@ --- "Composable index templates that include subobjects: auto on arbitrary field": - requires: - cluster_features: ["mapper.subobjects_auto"] - reason: "https://github.com/elastic/elasticsearch/issues/96768 fixed at 8.11.0" test_runner_features: "allowed_warnings" - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml index c88d638199dba..d07d03cb7146c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.sort/20_nested.yml @@ -1,8 +1,5 @@ --- sort doc with nested object: - - requires: - cluster_features: ["mapper.index_sorting_on_nested"] - reason: uses index sorting on nested fields - do: indices.create: index: test @@ -66,9 +63,6 @@ sort doc with nested object: --- sort doc on nested field: - - requires: - cluster_features: [ "mapper.index_sorting_on_nested" ] - reason: uses index sorting on nested fields - do: catch: /cannot apply index sort to field \[nested_field\.foo\] under nested object \[nested_field\]/ indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 07af3fb52b92f..2a31b3bd387c4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -312,7 +312,6 @@ override sort mode settings: --- override sort field using nested field type in sorting: - requires: - cluster_features: ["mapper.index_sorting_on_nested"] test_runner_features: [ capabilities ] capabilities: - method: PUT @@ -358,9 +357,6 @@ override sort field using nested field type in sorting: --- override sort field using nested field type: - - requires: - cluster_features: ["mapper.index_sorting_on_nested"] - reason: "Support for index sorting on indexes with nested objects required" - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index 084f104932d99..8485aba0ecc6a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -55,9 +55,6 @@ keyword: --- keyword with normalizer: - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: indices.create: index: test-keyword-with-normalizer diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 9d6e8da8c1e1e..2a14c291d5d31 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -417,7 +417,6 @@ - requires: test_runner_features: [arbitrary_key] - cluster_features: ["mapper.query_index_mode"] reason: "_ignored_source added to mappings" - do: @@ -511,10 +510,6 @@ --- "Lucene segment level fields stats": - - requires: - cluster_features: ["mapper.segment_level_fields_stats"] - reason: "segment level fields stats" - - do: indices.create: index: index1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml index 3ec854e93d82c..20e9d92a36088 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/90_fs_watermark_stats.yml @@ -1,8 +1,6 @@ --- "Allocation stats": - requires: - cluster_features: ["stats.include_disk_thresholds"] - reason: "fs watermark stats was added in 8.15.0" test_runner_features: [arbitrary_key] - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml index 3432a1e34c018..6ca17cc9cdce9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/range/10_basic.yml @@ -520,10 +520,6 @@ setup: --- "Null bounds": - - requires: - cluster_features: ["mapper.range.null_values_off_by_one_fix"] - reason: fixed in 8.15.0 - - do: index: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml index bd14fb182ac5a..94db54d152941 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: 'unified_highlighter_matched_fields' - reason: 'test requires unified highlighter to support matched_fields' - - do: indices.create: index: index1 diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml index a3d920d903ae8..bc4e262ea53c6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/101_knn_nested_search_bits.yml @@ -1,8 +1,6 @@ setup: - requires: - cluster_features: "mapper.vectors.bit_vectors" test_runner_features: close_to - reason: 'bit vectors added in 8.15' - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml index 3d4841a16d82d..cffc12a8d24ae 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/110_knn_query_with_filter.yml @@ -59,9 +59,6 @@ setup: --- "Simple knn query": - - requires: - cluster_features: "search.vectors.k_param_supported" - reason: 'k param for knn as query is required' - do: search: index: my_index diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml index f6538b573809a..c92c88df91641 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/190_knn_query-with-k-param.yml @@ -1,8 +1,6 @@ # test how knn query interact with other queries setup: - requires: - cluster_features: "search.vectors.k_param_supported" - reason: 'k param for knn as query is required' test_runner_features: close_to - do: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml index 3f81c0044d170..abde3e86dd05b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_bbq_hnsw.yml @@ -1,7 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.bbq" - reason: 'kNN float to better-binary quantization is required' - do: indices.create: index: bbq_hnsw diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml index baf568762dd17..9b27aea4b1db7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml @@ -1,7 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.int4_quantization" - reason: 'kNN float to half-byte quantization is required' - do: indices.create: index: hnsw_byte_quantized diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml index 0bc111576c2a9..2541de7023bf0 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_bbq_flat.yml @@ -1,7 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.bbq" - reason: 'kNN float to better-binary quantization is required' - do: indices.create: index: bbq_flat diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml index 0e0180e58fd96..f9f8d56e1d9c9 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml @@ -1,7 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.int4_quantization" - reason: 'kNN float to half-byte quantization is required' - do: indices.create: index: int4_flat diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml index 680433a5945fd..ef2ae3ba7ee0a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.bit_vectors" - reason: 'mapper.vectors.bit_vectors' - - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml index 783f08a5d4ff4..07261e6a30c77 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/45_knn_search_bit_flat.yml @@ -1,8 +1,4 @@ setup: - - requires: - cluster_features: "mapper.vectors.bit_vectors" - reason: 'mapper.vectors.bit_vectors' - - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml index 44d966b76f34e..8915325c3a67b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/330_fetch_fields.yml @@ -1128,10 +1128,6 @@ fetch geo_point: --- "Test with subobjects: auto": - - requires: - cluster_features: "mapper.subobjects_auto" - reason: requires support for subobjects auto setting - - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml index 1730a49f743d9..7e00cbb01c589 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml @@ -1,8 +1,5 @@ --- ignore_above mapping level setting: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -42,9 +39,6 @@ ignore_above mapping level setting: --- ignore_above mapping level setting on arrays: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -84,9 +78,6 @@ ignore_above mapping level setting on arrays: --- ignore_above mapping overrides setting: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -128,9 +119,6 @@ ignore_above mapping overrides setting: --- ignore_above mapping overrides setting on arrays: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -172,9 +160,6 @@ ignore_above mapping overrides setting on arrays: --- date ignore_above index level setting: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml index 772c3c24170cd..045f757b08302 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml @@ -5,9 +5,6 @@ setup: --- ignore_above mapping level setting: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -47,9 +44,6 @@ ignore_above mapping level setting: --- ignore_above mapping level setting on arrays: - - requires: - cluster_features: [ "mapper.flattened.ignore_above_with_arrays_support" ] - reason: requires support of ignore_above with arrays for flattened fields - do: indices.create: index: test @@ -90,9 +84,6 @@ ignore_above mapping level setting on arrays: --- ignore_above mapping overrides setting: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test @@ -135,9 +126,6 @@ ignore_above mapping overrides setting: --- ignore_above mapping overrides setting on arrays: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml index 3c29845871fe7..6e711ee143b06 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/550_ignore_above_invalid.yml @@ -16,9 +16,6 @@ ignore_above index setting negative value: --- keyword ignore_above mapping setting negative value: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: catch: bad_request indices.create: @@ -32,9 +29,6 @@ keyword ignore_above mapping setting negative value: --- flattened ignore_above mapping setting negative value: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: catch: bad_request indices.create: @@ -48,9 +42,6 @@ flattened ignore_above mapping setting negative value: --- wildcard ignore_above mapping setting negative value: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: catch: bad_request indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml index a4a9b1aaecb22..71e0c2d147c1e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/600_flattened_ignore_above.yml @@ -1,8 +1,5 @@ --- flattened ignore_above single-value field: - - requires: - cluster_features: [ "flattened.ignore_above_support" ] - reason: introduce ignore_above support in flattened fields - do: indices.create: index: test @@ -65,9 +62,6 @@ flattened ignore_above single-value field: --- flattened ignore_above multi-value field: - - requires: - cluster_features: [ "flattened.ignore_above_support" ] - reason: introduce ignore_above support in flattened fields - do: indices.create: index: test diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml index da0f00d960534..70a3b0253c78f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/simulate.ingest/10_basic.yml @@ -119,10 +119,6 @@ setup: - skip: features: headers - - requires: - cluster_features: ["simulate.mapping.validation"] - reason: "ingest simulate index mapping validation added in 8.16" - - do: headers: Content-Type: application/json @@ -265,10 +261,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.mapping.validation.templates"] - reason: "ingest simulate index mapping validation added in 8.16" - - do: indices.put_template: name: v1_template @@ -401,10 +393,6 @@ setup: - headers - allowed_warnings - - requires: - cluster_features: ["simulate.mapping.validation.templates"] - reason: "ingest simulate index mapping validation added in 8.16" - - do: allowed_warnings: - "index template [my-template1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template1] will take precedence during new index creation" diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 616afd3cf67ad..1e841c8893fc6 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -122,8 +122,6 @@ missing dimension on routing path field: multi-value routing path field succeeds: - requires: test_runner_features: close_to - cluster_features: ["routing.multi_value_routing_path"] - reason: support for multi-value dimensions - do: indices.create: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index beba6f2752a11..5a5ae03ab938f 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -65,9 +65,6 @@ setup: --- generates a consistent id: - - requires: - cluster_features: "tsdb.ts_routing_hash_doc_value_parse_byte_ref" - reason: _tsid routing hash doc value parsing has been fixed - do: bulk: refresh: true diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml index dae50704dd0d0..a8d256bbc097e 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/40_search.yml @@ -340,9 +340,6 @@ sort by tsid: --- aggs by index_mode: - - requires: - cluster_features: ["mapper.query_index_mode"] - reason: require _index_mode metadata field - do: search: index: test diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java index 420f6427a55e1..68e65b16aa3a2 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/IndicesRequestIT.java @@ -519,7 +519,7 @@ public void testDeleteIndex() { public void testGetMappings() { interceptTransportActions(GetMappingsAction.NAME); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(randomIndicesOrAliases()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(randomIndicesOrAliases()); internalCluster().coordOnlyNodeClient().admin().indices().getMappings(getMappingsRequest).actionGet(); clearInterceptedActions(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java index 4c25b3262d559..b14bf38f3cbcc 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateIndexIT.java @@ -120,7 +120,7 @@ public void testNonNestedMappings() throws Exception { ) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -130,7 +130,7 @@ public void testNonNestedMappings() throws Exception { public void testEmptyNestedMappings() throws Exception { assertAcked(prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().endObject())); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); @@ -150,7 +150,7 @@ public void testEmptyMappings() throws Exception { prepareCreate("test").setMapping(XContentFactory.jsonBuilder().startObject().startObject("_doc").endObject().endObject()) ); - GetMappingsResponse response = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); MappingMetadata mappings = response.mappings().get("test"); assertNotNull(mappings); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 867ca89f9e7f3..b9dadf86c3345 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -357,8 +357,9 @@ private void assertHasAliases(Set aliasNames, String name, String primar * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java index e45555b1dec19..2cd319d148321 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkIntegrationIT.java @@ -55,7 +55,7 @@ public void testBulkIndexCreatesMapping() throws Exception { bulkBuilder.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON); bulkBuilder.get(); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertTrue(mappingsResponse.getMappings().containsKey("logstash-2014.03.30")); }); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java index 7777dd778a6c1..9eed1f757b5b1 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/SimpleClusterStateIT.java @@ -258,11 +258,14 @@ public void testLargeClusterStatePublishing() throws Exception { .setTimeout(TimeValue.timeValueMinutes(1)) ); ensureGreen(); // wait for green state, so its both green, and there are no more pending events - MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings("test").get().getMappings().get("test"); + MappingMetadata masterMappingMetadata = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .getMappings() + .get("test"); for (Client client : clients()) { MappingMetadata mappingMetadata = client.admin() .indices() - .prepareGetMappings("test") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") .setLocal(true) .get() .getMappings() diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java index d6ccdf3dc0399..256566045c59a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java @@ -115,7 +115,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("integer_field") ); @@ -146,7 +146,7 @@ public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception { ) .get(); - getMappingsResponse = indicesAdmin().prepareGetMappings(index).get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); assertNotNull( ((Map) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("float_field") ); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java index 802ba04375c48..24af560f608d3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/HiddenIndexIT.java @@ -97,7 +97,7 @@ public void testGlobalTemplatesDoNotApply() { assertAcked(indicesAdmin().prepareCreate("a_hidden_index").setSettings(Settings.builder().put("index.hidden", true).build())); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings("a_hidden_index").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "a_hidden_index").get(); assertThat(mappingsResponse.mappings().size(), is(1)); MappingMetadata mappingMetadata = mappingsResponse.mappings().get("a_hidden_index"); assertNotNull(mappingMetadata); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java index 960ee2fd7ca60..6bca87ebd6e3d 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java @@ -49,7 +49,7 @@ public void testBasic() { assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest)); Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts"); assertThat(settings.get("index.mode"), equalTo("lookup")); - assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); + assertNull(settings.get("index.auto_expand_replicas")); Map allHosts = Map.of( "192.168.1.2", "Windows", @@ -141,7 +141,6 @@ public void testResizeLookupIndex() { Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2"); assertThat(settings.get("index.mode"), equalTo("lookup")); assertThat(settings.get("index.number_of_shards"), equalTo("1")); - assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all")); ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1"); split.setResizeType(ResizeType.SPLIT); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index a37fb25052ac5..06561bc6d4c97 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -96,7 +96,11 @@ public void testSimpleDynamicMappingsSuccessful() { client().prepareIndex("index").setId("1").setSource("a.x", 1).get(); client().prepareIndex("index").setId("2").setSource("a.y", 2).get(); - Map mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .mappings() + .get("index") + .sourceAsMap(); assertTrue(new WriteField("properties.a", () -> mappings).exists()); assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists()); } @@ -183,7 +187,7 @@ private Map indexConcurrently(int numberOfFieldsToCreate, Settin for (int i = 0; i < numberOfFieldsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); + GetMappingsResponse mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index").get(); MappingMetadata indexMappings = mappings.getMappings().get("index"); assertNotNull(indexMappings); Map typeMappingsMap = indexMappings.getSourceAsMap(); @@ -214,7 +218,11 @@ public void testConcurrentDynamicMappingsWithConflictingType() throws Throwable for (int i = 0; i < numberOfDocsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } - Map index = indicesAdmin().prepareGetMappings("index").get().getMappings().get("index").getSourceAsMap(); + Map index = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "index") + .get() + .getMappings() + .get("index") + .getSourceAsMap(); for (int i = 0, j = 1; i < numberOfDocsToCreate; i++, j++) { assertThat(new WriteField("properties.field" + i + ".type", () -> index).get(null), is(oneOf("long", "float"))); assertThat(new WriteField("properties.field" + j + ".type", () -> index).get(null), is(oneOf("long", "float"))); @@ -806,7 +814,11 @@ public void testSubobjectsFalseAtRoot() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); @SuppressWarnings("unchecked") Map properties = (Map) mappings.get("properties"); assertEquals(4, properties.size()); @@ -851,7 +863,11 @@ public void testSubobjectsFalse() throws Exception { assertEquals(RestStatus.CREATED, indexResponse.status()); assertBusy(() -> { - Map mappings = indicesAdmin().prepareGetMappings("test").get().mappings().get("test").sourceAsMap(); + Map mappings = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test") + .get() + .mappings() + .get("test") + .sourceAsMap(); Map properties = (Map) mappings.get("properties"); Map foo = (Map) properties.get("foo"); properties = (Map) foo.get("properties"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java index 4a7de4b0ebc23..1a51fc12fed8e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/MultiFieldsIntegrationIT.java @@ -37,7 +37,7 @@ public class MultiFieldsIntegrationIT extends ESIntegTestCase { public void testMultiFields() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createTypeSource())); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -53,7 +53,7 @@ public void testMultiFields() throws Exception { assertAcked(indicesAdmin().preparePutMapping("my-index").setSource(createPutMappingSource())); - getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); mappingSource = mappingMetadata.sourceAsMap(); @@ -74,7 +74,7 @@ public void testMultiFields() throws Exception { public void testGeoPointMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("geo_point"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -102,7 +102,7 @@ public void testGeoPointMultiField() throws Exception { public void testCompletionMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("completion"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); @@ -123,7 +123,7 @@ public void testCompletionMultiField() throws Exception { public void testIpMultiField() throws Exception { assertAcked(indicesAdmin().prepareCreate("my-index").setMapping(createMappingSource("ip"))); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("my-index").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "my-index").get(); MappingMetadata mappingMetadata = getMappingsResponse.mappings().get("my-index"); assertThat(mappingMetadata, not(nullValue())); Map mappingSource = mappingMetadata.sourceAsMap(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java index 652f4e02ffbce..c53cf3b56f65a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndicesOptionsIntegrationIT.java @@ -537,26 +537,26 @@ public void testPutMapping() throws Exception { } verify(indicesAdmin().preparePutMapping("foo").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); verify(indicesAdmin().preparePutMapping("b*").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("_all").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping().setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("foo").get().mappings().get("foo"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("foobar").get().mappings().get("foobar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("bar").get().mappings().get("bar"), notNullValue()); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foo").get().mappings().get("foo"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "foobar").get().mappings().get("foobar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "bar").get().mappings().get("bar"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); verify(indicesAdmin().preparePutMapping("c*").setSource("field", "type=text"), true); assertAcked(indicesAdmin().prepareClose("barbaz").get()); verify(indicesAdmin().preparePutMapping("barbaz").setSource("field", "type=text"), false); - assertThat(indicesAdmin().prepareGetMappings("barbaz").get().mappings().get("barbaz"), notNullValue()); + assertThat(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "barbaz").get().mappings().get("barbaz"), notNullValue()); } public static final class TestPlugin extends Plugin { @@ -664,7 +664,7 @@ static GetFieldMappingsRequestBuilder getFieldMapping(String... indices) { } static GetMappingsRequestBuilder getMapping(String... indices) { - return indicesAdmin().prepareGetMappings(indices); + return indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, indices); } static GetSettingsRequestBuilder getSettings(String... indices) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index de565605ff58a..7264585337fc7 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -123,8 +123,9 @@ private void triggerClusterStateUpdates() { * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. */ private void assertMappingsAndSettings(String expectedMappings) { - final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) - .actionGet(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) + ).actionGet(); final Map mappings = getMappingsResponse.getMappings(); assertThat( diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java index 71fcb25c2e0b2..e3092bda185fe 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java @@ -198,7 +198,7 @@ public void testGetFieldMappingsWithBlocks() throws Exception { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java index 20e59fab3bd0c..023aa402b7337 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/SimpleGetMappingsIT.java @@ -42,7 +42,7 @@ protected Collection> nodePlugins() { public void testGetMappingsWhereThereAreNone() { createIndex("index"); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().containsKey("index"), equalTo(true)); assertEquals(MappingMetadata.EMPTY_MAPPINGS, response.mappings().get("index")); } @@ -70,19 +70,19 @@ public void testSimpleGetMappings() throws Exception { assertThat(clusterHealth.isTimedOut(), equalTo(false)); // Get all mappings - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get all mappings, via wildcard support - response = indicesAdmin().prepareGetMappings("*").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "*").get(); assertThat(response.mappings().size(), equalTo(2)); assertThat(response.mappings().get("indexa"), notNullValue()); assertThat(response.mappings().get("indexb"), notNullValue()); // Get mappings in indexa - response = indicesAdmin().prepareGetMappings("indexa").get(); + response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "indexa").get(); assertThat(response.mappings().size(), equalTo(1)); assertThat(response.mappings().get("indexa"), notNullValue()); } @@ -94,7 +94,7 @@ public void testGetMappingsWithBlocks() throws IOException { for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) { try { enableIndexBlock("test", block); - GetMappingsResponse response = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertThat(response.mappings().size(), equalTo(1)); assertNotNull(response.mappings().get("test")); } finally { @@ -104,7 +104,7 @@ public void testGetMappingsWithBlocks() throws IOException { try { enableIndexBlock("test", SETTING_BLOCKS_METADATA); - assertBlocked(indicesAdmin().prepareGetMappings(), INDEX_METADATA_BLOCK); + assertBlocked(indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT), INDEX_METADATA_BLOCK); } finally { disableIndexBlock("test", SETTING_BLOCKS_METADATA); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java index 6f6e488d46b23..fa2598348a1ce 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationIT.java @@ -109,7 +109,7 @@ public void testUpdateMappingWithoutType() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"body":{"type":"text"},"date":{"type":"integer"}}}}""")); } @@ -123,7 +123,7 @@ public void testUpdateMappingWithoutTypeMultiObjects() { assertThat(putMappingResponse.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertThat(getMappingsResponse.mappings().get("test").source().toString(), equalTo(""" {"_doc":{"properties":{"date":{"type":"integer"}}}}""")); } @@ -215,7 +215,10 @@ public void testUpdateMappingConcurrently() throws Throwable { .get(); assertThat(response.isAcknowledged(), equalTo(true)); - GetMappingsResponse getMappingResponse = client2.admin().indices().prepareGetMappings(indexName).get(); + GetMappingsResponse getMappingResponse = client2.admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, indexName) + .get(); MappingMetadata mappings = getMappingResponse.getMappings().get(indexName); @SuppressWarnings("unchecked") Map properties = (Map) mappings.getSourceAsMap().get("properties"); @@ -284,7 +287,7 @@ private void assertConcreteMappingsOnAll(final String index, final String... fie * Waits for the given mapping type to exists on the master node. */ private void assertMappingOnMaster(final String index, final String... fieldNames) { - GetMappingsResponse response = indicesAdmin().prepareGetMappings(index).get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, index).get(); MappingMetadata mappings = response.getMappings().get(index); assertThat(mappings, notNullValue()); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 4112290fa4e04..2a68b65bcdccb 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -424,22 +424,14 @@ provides org.elasticsearch.features.FeatureSpecification with - org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures, org.elasticsearch.action.bulk.BulkFeatures, org.elasticsearch.features.FeatureInfrastructureFeatures, - org.elasticsearch.health.HealthFeatures, - org.elasticsearch.cluster.metadata.MetadataFeatures, - org.elasticsearch.rest.RestFeatures, - org.elasticsearch.repositories.RepositoriesFeatures, - org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures, org.elasticsearch.index.mapper.MapperFeatures, org.elasticsearch.index.IndexFeatures, - org.elasticsearch.ingest.IngestGeoIpFeatures, org.elasticsearch.search.SearchFeatures, org.elasticsearch.script.ScriptFeatures, org.elasticsearch.search.retriever.RetrieversFeatures, - org.elasticsearch.reservedstate.service.FileSettingsFeatures, org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures; uses org.elasticsearch.plugins.internal.SettingsExtension; diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 98d6284fd91d2..ec393b7af5cdf 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -1004,7 +1004,7 @@ public void initRestHandlers(Supplier nodesInCluster, Predicate< // Desired nodes registerHandler.accept(new RestGetDesiredNodesAction()); - registerHandler.accept(new RestUpdateDesiredNodesAction(clusterSupportsFeature)); + registerHandler.accept(new RestUpdateDesiredNodesAction()); registerHandler.accept(new RestDeleteDesiredNodesAction()); for (ActionPlugin plugin : actionPlugins) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java deleted file mode 100644 index 164fc816ad367..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/AllocationStatsFeatures.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.cluster.allocation; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class AllocationStatsFeatures implements FeatureSpecification { - public static final NodeFeature INCLUDE_DISK_THRESHOLD_SETTINGS = new NodeFeature("stats.include_disk_thresholds", true); - - @Override - public Set getFeatures() { - return Set.of(INCLUDE_DISK_THRESHOLD_SETTINGS); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java index d929fb457d5d1..23bf22e08985e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsAction.java @@ -30,7 +30,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -49,7 +48,6 @@ public class TransportGetAllocationStatsAction extends TransportMasterNodeReadAc private final AllocationStatsService allocationStatsService; private final DiskThresholdSettings diskThresholdSettings; - private final FeatureService featureService; @Inject public TransportGetAllocationStatsAction( @@ -58,8 +56,7 @@ public TransportGetAllocationStatsAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - AllocationStatsService allocationStatsService, - FeatureService featureService + AllocationStatsService allocationStatsService ) { super( TYPE.name(), @@ -74,7 +71,6 @@ public TransportGetAllocationStatsAction( ); this.allocationStatsService = allocationStatsService; this.diskThresholdSettings = new DiskThresholdSettings(clusterService.getSettings(), clusterService.getClusterSettings()); - this.featureService = featureService; } @Override @@ -92,10 +88,7 @@ protected void masterOperation(Task task, Request request, ClusterState state, A listener.onResponse( new Response( request.metrics().contains(Metric.ALLOCATIONS) ? allocationStatsService.stats() : Map.of(), - request.metrics().contains(Metric.FS) - && featureService.clusterHasFeature(clusterService.state(), AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS) - ? diskThresholdSettings - : null + request.metrics().contains(Metric.FS) ? diskThresholdSettings : null ) ); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java index 9fede2ebb5be6..beb0e1f927de2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/capabilities/TransportNodesCapabilitiesAction.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.cluster.node.capabilities; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -20,11 +19,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequest; @@ -32,7 +29,6 @@ import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Set; public class TransportNodesCapabilitiesAction extends TransportNodesAction< @@ -45,7 +41,6 @@ public class TransportNodesCapabilitiesAction extends TransportNodesAction< public static final ActionType TYPE = new ActionType<>("cluster:monitor/nodes/capabilities"); private final RestController restController; - private final FeatureService featureService; @Inject public TransportNodesCapabilitiesAction( @@ -53,8 +48,7 @@ public TransportNodesCapabilitiesAction( ClusterService clusterService, TransportService transportService, ActionFilters actionFilters, - RestController restController, - FeatureService featureService + RestController restController ) { super( TYPE.name(), @@ -65,23 +59,6 @@ public TransportNodesCapabilitiesAction( threadPool.executor(ThreadPool.Names.MANAGEMENT) ); this.restController = restController; - this.featureService = featureService; - } - - @Override - protected void doExecute(Task task, NodesCapabilitiesRequest request, ActionListener listener) { - if (featureService.clusterHasFeature(clusterService.state(), RestNodesCapabilitiesAction.CAPABILITIES_ACTION) == false) { - // not everything in the cluster supports capabilities. - // Therefore we don't support whatever it is we're being asked for - listener.onResponse(new NodesCapabilitiesResponse(clusterService.getClusterName(), List.of(), List.of()) { - @Override - public Optional isSupported() { - return Optional.of(false); - } - }); - } else { - super.doExecute(task, request, listener); - } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java index 7e3c38c735091..ca02d19749ae7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsAction.java @@ -13,13 +13,18 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.UpdateForV10; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; public class ClusterGetSettingsAction extends ActionType { @@ -34,25 +39,29 @@ public ClusterGetSettingsAction() { /** * Request to retrieve the cluster settings */ - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { public Request(TimeValue masterNodeTimeout) { super(masterNodeTimeout); } + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) public Request(StreamInput in) throws IOException { super(in); assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); } @Override - public void writeTo(StreamOutput out) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - super.writeTo(out); + public ActionRequestValidationException validate() { + return null; } @Override - public ActionRequestValidationException validate() { - return null; + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); } } @@ -79,20 +88,17 @@ public int hashCode() { return Objects.hash(persistentSettings, transientSettings, settings); } - public Response(StreamInput in) throws IOException { - super(in); - assert in.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); - persistentSettings = Settings.readSettingsFromStream(in); - transientSettings = Settings.readSettingsFromStream(in); - settings = Settings.readSettingsFromStream(in); - } - public Response(Settings persistentSettings, Settings transientSettings, Settings settings) { this.persistentSettings = Objects.requireNonNullElse(persistentSettings, Settings.EMPTY); this.transientSettings = Objects.requireNonNullElse(transientSettings, Settings.EMPTY); this.settings = Objects.requireNonNullElse(settings, Settings.EMPTY); } + /** + * NB prior to 9.0 get-component was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) @Override public void writeTo(StreamOutput out) throws IOException { assert out.getTransportVersion().onOrAfter(TransportVersions.V_8_3_0); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java index dce6a38001392..71b976e012aad 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/settings/TransportClusterGetSettingsAction.java @@ -11,57 +11,66 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -public class TransportClusterGetSettingsAction extends TransportMasterNodeReadAction< +public class TransportClusterGetSettingsAction extends TransportLocalClusterStateAction< ClusterGetSettingsAction.Request, ClusterGetSettingsAction.Response> { private final SettingsFilter settingsFilter; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) + @SuppressWarnings("this-escape") @Inject public TransportClusterGetSettingsAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, SettingsFilter settingsFilter, - ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver + ActionFilters actionFilters ) { super( ClusterGetSettingsAction.NAME, - false, - transportService, - clusterService, - threadPool, actionFilters, - ClusterGetSettingsAction.Request::new, - indexNameExpressionResolver, - ClusterGetSettingsAction.Response::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); - this.settingsFilter = settingsFilter; + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + ClusterGetSettingsAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, ClusterGetSettingsAction.Request request, ClusterState state, ActionListener listener ) throws Exception { + ((CancellableTask) task).ensureNotCancelled(); Metadata metadata = state.metadata(); listener.onResponse( new ClusterGetSettingsAction.Response( diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index 1bc2e1d13c864..29a124b3d0b20 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -21,7 +21,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -71,7 +71,7 @@ public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { } AnalysisStats.countMapping(mappingCounts, indexMetadata); - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()); sourceModeUsageCount.merge(sourceMode.toString().toLowerCase(Locale.ENGLISH), 1, Integer::sum); } final AtomicLong totalFieldCount = new AtomicLong(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java index dd4114c947174..84789d8a2acfb 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequest.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; @@ -21,7 +22,9 @@ public class GetMappingsRequest extends ClusterInfoRequest { - public GetMappingsRequest() {} + public GetMappingsRequest(TimeValue masterTimeout) { + super(masterTimeout); + } public GetMappingsRequest(StreamInput in) throws IOException { super(in); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java index 3f54138581398..a12ba4f60c26a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsRequestBuilder.java @@ -11,13 +11,14 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequestBuilder; import org.elasticsearch.client.internal.ElasticsearchClient; +import org.elasticsearch.core.TimeValue; public class GetMappingsRequestBuilder extends ClusterInfoRequestBuilder< GetMappingsRequest, GetMappingsResponse, GetMappingsRequestBuilder> { - public GetMappingsRequestBuilder(ElasticsearchClient client, String... indices) { - super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest().indices(indices)); + public GetMappingsRequestBuilder(ElasticsearchClient client, TimeValue masterTimeout, String... indices) { + super(client, GetMappingsAction.INSTANCE, new GetMappingsRequest(masterTimeout).indices(indices)); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java index be08293fe90db..c845d1a3854c9 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/LazyRolloverAction.java @@ -33,7 +33,6 @@ import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; @@ -55,8 +54,6 @@ public final class LazyRolloverAction extends ActionType { private static final Logger logger = LogManager.getLogger(LazyRolloverAction.class); - public static final NodeFeature DATA_STREAM_LAZY_ROLLOVER = new NodeFeature("data_stream.rollover.lazy", true); - public static final LazyRolloverAction INSTANCE = new LazyRolloverAction(); public static final String NAME = "indices:admin/data_stream/lazy_rollover"; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java index 67b6df150c458..6106e620521f7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndexStats.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import java.util.ArrayList; import java.util.HashMap; @@ -22,9 +21,6 @@ public class IndexStats implements Iterable { - // feature was effectively reverted but we still need to keep this constant around - public static final NodeFeature REVERTED_TIER_CREATION_DATE = new NodeFeature("stats.tier_creation_date", true); - private final String index; private final String uuid; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java deleted file mode 100644 index 558343db1023a..0000000000000 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsFeatures.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.indices.stats; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class IndicesStatsFeatures implements FeatureSpecification { - - @Override - public Set getFeatures() { - return Set.of(IndexStats.REVERTED_TIER_CREATION_DATE); - } -} diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java index ce29d65ececf9..003be58d19554 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequest.java @@ -12,16 +12,20 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; -public class SimulateIndexTemplateRequest extends MasterNodeReadRequest { +public class SimulateIndexTemplateRequest extends LocalClusterStateRequest { private String indexName; @@ -30,14 +34,18 @@ public class SimulateIndexTemplateRequest extends MasterNodeReadRequest headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + public String getIndexName() { return indexName; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java index a521dac60e96a..1a04b6e4d7633 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateResponse.java @@ -12,13 +12,11 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; -import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention; import org.elasticsearch.cluster.metadata.ResettableValue; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; @@ -67,27 +65,11 @@ public RolloverConfiguration getRolloverConfiguration() { return rolloverConfiguration; } - public SimulateIndexTemplateResponse(StreamInput in) throws IOException { - super(in); - resolvedTemplate = in.readOptionalWriteable(Template::new); - if (in.readBoolean()) { - int overlappingTemplatesCount = in.readInt(); - overlappingTemplates = Maps.newMapWithExpectedSize(overlappingTemplatesCount); - for (int i = 0; i < overlappingTemplatesCount; i++) { - String templateName = in.readString(); - overlappingTemplates.put(templateName, in.readStringCollectionAsList()); - } - } else { - this.overlappingTemplates = null; - } - rolloverConfiguration = in.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X) - ? in.readOptionalWriteable(RolloverConfiguration::new) - : null; - if (in.getTransportVersion().between(TransportVersions.V_8_14_0, TransportVersions.V_8_16_0)) { - in.readOptionalWriteable(DataStreamGlobalRetention::read); - } - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) @Override public void writeTo(StreamOutput out) throws IOException { out.writeOptionalWriteable(resolvedTemplate); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java index 75cc72416a854..15015b910767e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateAction.java @@ -14,12 +14,16 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.local.LocalClusterStateRequest; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; import java.util.Objects; /** @@ -35,7 +39,7 @@ private SimulateTemplateAction() { super(NAME); } - public static class Request extends MasterNodeReadRequest { + public static class Request extends LocalClusterStateRequest { @Nullable private String templateName; @@ -44,26 +48,15 @@ public static class Request extends MasterNodeReadRequest { private TransportPutComposableIndexTemplateAction.Request indexTemplateRequest; private boolean includeDefaults = false; - public Request() { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - } - - public Request(String templateName) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (templateName == null) { - throw new IllegalArgumentException("template name cannot be null"); - } + public Request(TimeValue masterTimeout, String templateName) { + super(masterTimeout); this.templateName = templateName; } - public Request(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { - super(TRAPPY_IMPLICIT_DEFAULT_MASTER_NODE_TIMEOUT); - if (indexTemplateRequest == null) { - throw new IllegalArgumentException("index template body must be present"); - } - this.indexTemplateRequest = indexTemplateRequest; - } - + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until + * we no longer need to support calling this action remotely. + */ public Request(StreamInput in) throws IOException { super(in); templateName = in.readOptionalString(); @@ -73,16 +66,6 @@ public Request(StreamInput in) throws IOException { } } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalString(templateName); - out.writeOptionalWriteable(indexTemplateRequest); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_9_X)) { - out.writeBoolean(includeDefaults); - } - } - @Override public ActionRequestValidationException validate() { ActionRequestValidationException validationException = null; @@ -98,6 +81,11 @@ public ActionRequestValidationException validate() { return validationException; } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + @Nullable public String getTemplateName() { return templateName; @@ -112,11 +100,6 @@ public TransportPutComposableIndexTemplateAction.Request getIndexTemplateRequest return indexTemplateRequest; } - public Request templateName(String templateName) { - this.templateName = templateName; - return this; - } - public Request indexTemplateRequest(TransportPutComposableIndexTemplateAction.Request indexTemplateRequest) { this.indexTemplateRequest = indexTemplateRequest; return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d3d557b598b3a..74936128caa25 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -11,7 +11,8 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; @@ -20,7 +21,6 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateIndexService; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; @@ -31,6 +31,7 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; @@ -42,7 +43,6 @@ import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -65,7 +65,7 @@ import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveLifecycle; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; -public class TransportSimulateIndexTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateIndexTemplateAction extends TransportLocalClusterStateAction< SimulateIndexTemplateRequest, SimulateIndexTemplateResponse> { @@ -77,14 +77,18 @@ public class TransportSimulateIndexTemplateAction extends TransportMasterNodeRea private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateIndexTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -92,13 +96,9 @@ public TransportSimulateIndexTemplateAction( ) { super( SimulateIndexTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateIndexTemplateRequest::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -108,10 +108,19 @@ public TransportSimulateIndexTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateIndexTemplateRequest::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateIndexTemplateRequest request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 30bbad0b57df0..692f027b23f9e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -11,26 +11,26 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.action.support.ChannelActionListener; +import org.elasticsearch.action.support.local.TransportLocalClusterStateAction; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStreamLifecycle; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.core.UpdateForV10; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettingProviders; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; -import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.NamedXContentRegistry; @@ -48,7 +48,7 @@ * Handles simulating an index template either by name (looking it up in the * cluster state), or by a provided template configuration */ -public class TransportSimulateTemplateAction extends TransportMasterNodeReadAction< +public class TransportSimulateTemplateAction extends TransportLocalClusterStateAction< SimulateTemplateAction.Request, SimulateIndexTemplateResponse> { @@ -60,14 +60,18 @@ public class TransportSimulateTemplateAction extends TransportMasterNodeReadActi private final ClusterSettings clusterSettings; private final boolean isDslOnlyMode; + /** + * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC it must be registered with the TransportService until + * we no longer need to support calling this action remotely. + */ + @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT) + @SuppressWarnings("this-escape") @Inject public TransportSimulateTemplateAction( TransportService transportService, ClusterService clusterService, - ThreadPool threadPool, MetadataIndexTemplateService indexTemplateService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, NamedXContentRegistry xContentRegistry, IndicesService indicesService, SystemIndices systemIndices, @@ -75,13 +79,9 @@ public TransportSimulateTemplateAction( ) { super( SimulateTemplateAction.NAME, - transportService, - clusterService, - threadPool, actionFilters, - SimulateTemplateAction.Request::new, - indexNameExpressionResolver, - SimulateIndexTemplateResponse::new, + transportService.getTaskManager(), + clusterService, EsExecutors.DIRECT_EXECUTOR_SERVICE ); this.indexTemplateService = indexTemplateService; @@ -91,10 +91,19 @@ public TransportSimulateTemplateAction( this.indexSettingProviders = indexSettingProviders.getIndexSettingProviders(); this.clusterSettings = clusterService.getClusterSettings(); this.isDslOnlyMode = isDataStreamsLifecycleOnlyMode(clusterService.getSettings()); + + transportService.registerRequestHandler( + actionName, + executor, + false, + true, + SimulateTemplateAction.Request::new, + (request, channel, task) -> executeDirect(task, request, new ChannelActionListener<>(channel)) + ); } @Override - protected void masterOperation( + protected void localClusterStateOperation( Task task, SimulateTemplateAction.Request request, ClusterState state, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index 998a3ada5d157..5851549977eab 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -14,24 +14,10 @@ import java.util.Set; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_IGNORED_FIELDS; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_ADDITION; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; -import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of( - SIMULATE_MAPPING_VALIDATION, - SIMULATE_MAPPING_VALIDATION_TEMPLATES, - SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS, - SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS, - SIMULATE_MAPPING_ADDITION, - SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING, - SIMULATE_IGNORED_FIELDS - ); + return Set.of(SIMULATE_IGNORED_FIELDS); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 2a6a789d9d312..523381321ada7 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -44,7 +44,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; import org.elasticsearch.indices.SystemIndices; @@ -80,7 +79,6 @@ public class TransportBulkAction extends TransportAbstractBulkAction { private static final Logger logger = LogManager.getLogger(TransportBulkAction.class); public static final String LAZY_ROLLOVER_ORIGIN = "lazy_rollover"; - private final FeatureService featureService; private final NodeClient client; private final IndexNameExpressionResolver indexNameExpressionResolver; private final OriginSettingClient rolloverClient; @@ -93,7 +91,6 @@ public TransportBulkAction( TransportService transportService, ClusterService clusterService, IngestService ingestService, - FeatureService featureService, NodeClient client, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, @@ -107,7 +104,6 @@ public TransportBulkAction( transportService, clusterService, ingestService, - featureService, client, actionFilters, indexNameExpressionResolver, @@ -124,7 +120,6 @@ public TransportBulkAction( TransportService transportService, ClusterService clusterService, IngestService ingestService, - FeatureService featureService, NodeClient client, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, @@ -141,7 +136,6 @@ public TransportBulkAction( transportService, clusterService, ingestService, - featureService, client, actionFilters, indexNameExpressionResolver, @@ -160,7 +154,6 @@ public TransportBulkAction( TransportService transportService, ClusterService clusterService, IngestService ingestService, - FeatureService featureService, NodeClient client, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, @@ -184,7 +177,6 @@ public TransportBulkAction( ); this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings; Objects.requireNonNull(relativeTimeProvider); - this.featureService = featureService; this.client = client; this.indexNameExpressionResolver = indexNameExpressionResolver; this.rolloverClient = new OriginSettingClient(client, LAZY_ROLLOVER_ORIGIN); @@ -283,7 +275,6 @@ private void populateMissingTargets( // A map for memorizing which indices exist. Map indexExistence = new HashMap<>(); Function indexExistenceComputation = (index) -> indexNameExpressionResolver.hasIndexAbstraction(index, state); - boolean lazyRolloverFeature = featureService.clusterHasFeature(state, LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER); boolean lazyRolloverFailureStoreFeature = DataStream.isFailureStoreFeatureFlagEnabled(); Set indicesThatRequireAlias = new HashSet<>(); @@ -328,18 +319,15 @@ private void populateMissingTargets( } } // Determine which data streams and failure stores need to be rolled over. - if (lazyRolloverFeature) { - DataStream dataStream = state.metadata().dataStreams().get(request.index()); - if (dataStream != null) { - if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) { - dataStreamsToBeRolledOver.add(request.index()); - } else if (lazyRolloverFailureStoreFeature - && writeToFailureStore - && dataStream.getFailureIndices().isRolloverOnWrite()) { - failureStoresToBeRolledOver.add(request.index()); - } + DataStream dataStream = state.metadata().dataStreams().get(request.index()); + if (dataStream != null) { + if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) { + dataStreamsToBeRolledOver.add(request.index()); + } else if (lazyRolloverFailureStoreFeature && writeToFailureStore && dataStream.getFailureIndices().isRolloverOnWrite()) { + failureStoresToBeRolledOver.add(request.index()); } } + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 2d65bea4ac5c2..18c420d99f525 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -83,15 +83,6 @@ * shards are not actually modified). */ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { - public static final NodeFeature SIMULATE_MAPPING_VALIDATION = new NodeFeature("simulate.mapping.validation", true); - public static final NodeFeature SIMULATE_MAPPING_VALIDATION_TEMPLATES = new NodeFeature("simulate.mapping.validation.templates", true); - public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( - "simulate.component.template.substitutions", - true - ); - public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions", true); - public static final NodeFeature SIMULATE_MAPPING_ADDITION = new NodeFeature("simulate.mapping.addition", true); - public static final NodeFeature SIMULATE_SUPPORT_NON_TEMPLATE_MAPPING = new NodeFeature("simulate.support.non.template.mapping", true); public static final NodeFeature SIMULATE_IGNORED_FIELDS = new NodeFeature("simulate.ignored.fields"); private final IndicesService indicesService; diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java index 578b0a61aafde..2eed45e5afa6d 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingService.java @@ -23,8 +23,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.Index; import java.util.List; @@ -43,8 +41,6 @@ public class DataStreamAutoShardingService { private static final Logger logger = LogManager.getLogger(DataStreamAutoShardingService.class); public static final String DATA_STREAMS_AUTO_SHARDING_ENABLED = "data_streams.auto_sharding.enabled"; - public static final NodeFeature DATA_STREAM_AUTO_SHARDING_FEATURE = new NodeFeature("data_stream.auto_sharding", true); - public static final Setting> DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING = Setting.listSetting( "data_streams.auto_sharding.excludes", List.of(), @@ -101,7 +97,6 @@ public class DataStreamAutoShardingService { ); private final ClusterService clusterService; private final boolean isAutoShardingEnabled; - private final FeatureService featureService; private final LongSupplier nowSupplier; private volatile TimeValue increaseShardsCooldown; private volatile TimeValue reduceShardsCooldown; @@ -109,12 +104,7 @@ public class DataStreamAutoShardingService { private volatile int maxWriteThreads; private volatile List dataStreamExcludePatterns; - public DataStreamAutoShardingService( - Settings settings, - ClusterService clusterService, - FeatureService featureService, - LongSupplier nowSupplier - ) { + public DataStreamAutoShardingService(Settings settings, ClusterService clusterService, LongSupplier nowSupplier) { this.clusterService = clusterService; this.isAutoShardingEnabled = settings.getAsBoolean(DATA_STREAMS_AUTO_SHARDING_ENABLED, false); this.increaseShardsCooldown = DATA_STREAMS_AUTO_SHARDING_INCREASE_SHARDS_COOLDOWN.get(settings); @@ -122,7 +112,6 @@ public DataStreamAutoShardingService( this.minWriteThreads = CLUSTER_AUTO_SHARDING_MIN_WRITE_THREADS.get(settings); this.maxWriteThreads = CLUSTER_AUTO_SHARDING_MAX_WRITE_THREADS.get(settings); this.dataStreamExcludePatterns = DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING.get(settings); - this.featureService = featureService; this.nowSupplier = nowSupplier; } @@ -168,15 +157,6 @@ public AutoShardingResult calculate(ClusterState state, DataStream dataStream, @ return NOT_APPLICABLE_RESULT; } - if (featureService.clusterHasFeature(state, DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE) == false) { - logger.debug( - "Data stream auto sharding service cannot compute the optimal number of shards for data stream [{}] because the cluster " - + "doesn't have the auto sharding feature", - dataStream.getName() - ); - return NOT_APPLICABLE_RESULT; - } - if (dataStreamExcludePatterns.stream().anyMatch(pattern -> Regex.simpleMatch(pattern, dataStream.getName()))) { logger.debug( "Data stream [{}] is excluded from auto sharding via the [{}] setting", diff --git a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java index 943b03588d4ba..634a103e9754a 100644 --- a/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/master/info/ClusterInfoRequest.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; @@ -27,11 +28,22 @@ public abstract class ClusterInfoRequest getMappings(GetMappingsRequest request) return execute(GetMappingsAction.INSTANCE, request); } - public GetMappingsRequestBuilder prepareGetMappings(String... indices) { - return new GetMappingsRequestBuilder(this, indices); + public GetMappingsRequestBuilder prepareGetMappings(TimeValue masterTimeout, String... indices) { + return new GetMappingsRequestBuilder(this, masterTimeout, indices); } public void getFieldMappings(GetFieldMappingsRequest request, ActionListener listener) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java index 673960c713391..17267525d4bdf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStreamGlobalRetention.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.NodeFeature; import java.io.IOException; @@ -23,7 +22,6 @@ */ public record DataStreamGlobalRetention(@Nullable TimeValue defaultRetention, @Nullable TimeValue maxRetention) implements Writeable { - public static final NodeFeature GLOBAL_RETENTION = new NodeFeature("data_stream.lifecycle.global_retention", true); public static final TimeValue MIN_RETENTION_VALUE = TimeValue.timeValueSeconds(10); /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java index 353f17fe0e00c..8366083b1907e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.unit.Processors; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; @@ -46,9 +45,6 @@ public final class DesiredNode implements Writeable, ToXContentObject, Comparable { - public static final NodeFeature RANGE_FLOAT_PROCESSORS_SUPPORTED = new NodeFeature("desired_node.range_float_processors"); - public static final NodeFeature DESIRED_NODE_VERSION_DEPRECATED = new NodeFeature("desired_node.version_deprecated", true); - public static final TransportVersion RANGE_FLOAT_PROCESSORS_SUPPORT_TRANSPORT_VERSION = TransportVersions.V_8_3_0; private static final ParseField SETTINGS_FIELD = new ParseField("settings"); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 7fc8a8693dcdc..e24d921ba78fd 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -69,7 +69,6 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.IndexCreationException; import org.elasticsearch.indices.IndicesService; @@ -237,6 +236,9 @@ public SystemIndices getSystemIndices() { * Validate the name for an index or alias against some static rules. */ public static void validateIndexOrAliasName(String index, BiFunction exceptionCtor) { + if (index == null || index.isEmpty()) { + throw exceptionCtor.apply(index, "must not be empty"); + } if (Strings.validFileName(index) == false) { throw exceptionCtor.apply(index, "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); } @@ -1583,7 +1585,7 @@ static void validateCloneIndex( private static final Set UNMODIFIABLE_SETTINGS_DURING_RESIZE = Set.of( IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java deleted file mode 100644 index 49bd38330e3af..0000000000000 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.cluster.metadata; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class MetadataFeatures implements FeatureSpecification { - @Override - public Set getFeatures() { - return Set.of(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED); - } -} diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index e38cd677991f3..24b14a46c8782 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -54,8 +54,6 @@ */ public abstract class IndexRouting { - static final NodeFeature BOOLEAN_ROUTING_PATH = new NodeFeature("routing.boolean_routing_path", true); - static final NodeFeature MULTI_VALUE_ROUTING_PATH = new NodeFeature("routing.multi_value_routing_path", true); static final NodeFeature LOGSB_ROUTE_ON_SORT_FIELDS = new NodeFeature("routing.logsb_route_on_sort_fields"); /** diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java index 1545fdf90d111..461ac50e1efc8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingFeatures.java @@ -18,7 +18,7 @@ public class RoutingFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(IndexRouting.BOOLEAN_ROUTING_PATH, IndexRouting.MULTI_VALUE_ROUTING_PATH); + return Set.of(); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 5d1e6741c5e22..2f2fd4ef453f6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -158,9 +158,7 @@ public ClusterState applyStartedShards(ClusterState clusterState, List ass * The placeholder value for {@link DesiredBalance} when the node stands down as master. */ public static final DesiredBalance NOT_MASTER = new DesiredBalance(-2, Map.of()); + /** * The starting value for {@link DesiredBalance} when the node becomes the master. */ @@ -57,6 +65,10 @@ public static boolean hasChanges(DesiredBalance a, DesiredBalance b) { return Objects.equals(a.assignments, b.assignments) == false; } + /** + * Returns the sum of shard movements needed to reach the new desired balance. Doesn't count new shard copies as a move, nor removal or + * unassignment of a shard copy. + */ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { var intersection = Sets.intersection(old.assignments().keySet(), updated.assignments().keySet()); int movements = 0; @@ -70,8 +82,15 @@ public static int shardMovements(DesiredBalance old, DesiredBalance updated) { return movements; } + /** + * Returns the number of shard movements needed to reach the new shard assignment. Doesn't count new shard copies as a move, nor removal + * or unassignment of a shard copy. + */ private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - var movements = Math.min(0, old.assigned() - updated.assigned());// compensate newly started shards + // A shard move should retain the same number of assigned nodes, just swap out one node for another. We will compensate for newly + // started shards -- adding a shard copy is not a move -- by initializing the count with a negative value so that incrementing later + // for a new node zeros out. + var movements = Math.min(0, old.assigned() - updated.assigned()); for (String nodeId : updated.nodeIds()) { if (old.nodeIds().contains(nodeId) == false) { movements++; diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java index 3b22221ea7db4..03630c284fa30 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceComputer.java @@ -415,11 +415,14 @@ boolean hasEnoughIterations(int currentIteration) { } private static Map collectShardAssignments(RoutingNodes routingNodes) { - final var entries = routingNodes.getAssignedShards().entrySet(); - assert entries.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; - final Map res = Maps.newHashMapWithExpectedSize(entries.size()); - for (var shardAndAssignments : entries) { - res.put(shardAndAssignments.getKey(), ShardAssignment.ofAssignedShards(shardAndAssignments.getValue())); + final var allAssignedShards = routingNodes.getAssignedShards().entrySet(); + assert allAssignedShards.stream().flatMap(t -> t.getValue().stream()).allMatch(ShardRouting::started) : routingNodes; + final Map res = Maps.newHashMapWithExpectedSize(allAssignedShards.size()); + for (var shardIdAndShardRoutings : allAssignedShards) { + res.put( + shardIdAndShardRoutings.getKey(), + ShardAssignment.createFromAssignedShardRoutingsList(shardIdAndShardRoutings.getValue()) + ); } return res; } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java index 83b370c1a7928..909a7a7a99a61 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceReconciler.java @@ -54,6 +54,10 @@ public class DesiredBalanceReconciler { private static final Logger logger = LogManager.getLogger(DesiredBalanceReconciler.class); + /** + * The minimum interval that log messages will be written if the number of undesired shard allocations reaches the percentage of total + * shards set by {@link #UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING}. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING = Setting.timeSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.log_interval", TimeValue.timeValueHours(1), @@ -62,6 +66,10 @@ public class DesiredBalanceReconciler { Setting.Property.NodeScope ); + /** + * Warning log messages may be periodically written if the number of shards that are on undesired nodes reaches this percentage setting. + * Works together with {@link #UNDESIRED_ALLOCATIONS_LOG_INTERVAL_SETTING} to log on a periodic basis. + */ public static final Setting UNDESIRED_ALLOCATIONS_LOG_THRESHOLD_SETTING = Setting.doubleSetting( "cluster.routing.allocation.desired_balance.undesired_allocations.threshold", 0.1, @@ -96,6 +104,13 @@ public DesiredBalanceReconciler( this.nodeAllocationStatsAndWeightsCalculator = nodeAllocationStatsAndWeightsCalculator; } + /** + * Applies a desired shard allocation to the routing table by initializing and relocating shards in the cluster state. + * + * @param desiredBalance The new desired cluster shard allocation + * @param allocation Cluster state information with which to make decisions, contains routing table metadata that will be modified to + * reach the given desired balance. + */ public void reconcile(DesiredBalance desiredBalance, RoutingAllocation allocation) { var nodeIds = allocation.routingNodes().getAllNodeIds(); allocationOrdering.retainNodes(nodeIds); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java index 5be26f0b3e8c7..2c73a27ad3418 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceShardsAllocator.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.metadata.SingleNodeShutdownMetadata; import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.AllocationService; import org.elasticsearch.cluster.routing.allocation.AllocationService.RerouteStrategy; import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsAndWeightsCalculator; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; @@ -56,11 +57,30 @@ public class DesiredBalanceShardsAllocator implements ShardsAllocator { private final ShardsAllocator delegateAllocator; private final ThreadPool threadPool; + /** + * This is a callback to run {@link AllocationService#executeWithRoutingAllocation(ClusterState, String, RerouteStrategy)}, which + * produces a new ClusterState with the changes made by {@link DesiredBalanceReconciler#reconcile}. The {@link RerouteStrategy} provided + * to the callback calls into {@link #desiredBalanceReconciler} for the changes. The {@link #masterServiceTaskQueue} will publish the + * new cluster state after the cluster state is constructed by the {@link ReconcileDesiredBalanceExecutor}. + */ private final DesiredBalanceReconcilerAction reconciler; private final DesiredBalanceComputer desiredBalanceComputer; + /** + * Reconciliation ({@link DesiredBalanceReconciler#reconcile(DesiredBalance, RoutingAllocation)}) takes the {@link DesiredBalance} + * output of {@link DesiredBalanceComputer#compute} and identifies how shards need to be added, moved or removed to go from the current + * cluster shard allocation to the new desired allocation. + */ private final DesiredBalanceReconciler desiredBalanceReconciler; private final ContinuousComputation desiredBalanceComputation; - private final PendingListenersQueue queue; + /** + * Saves and runs listeners after DesiredBalance computations complete. + */ + private final PendingListenersQueue pendingListenersQueue; + /** + * Each reroute request gets assigned a monotonically increasing sequence number. Many reroute requests may arrive before the balancer + * asynchronously runs a computation. The balancer will use the latest request and save this sequence number to track back to the + * request. + */ private final AtomicLong indexGenerator = new AtomicLong(-1); private final ConcurrentLinkedQueue> pendingDesiredBalanceMoves = new ConcurrentLinkedQueue<>(); private final MasterServiceTaskQueue masterServiceTaskQueue; @@ -199,7 +219,7 @@ public String toString() { return "DesiredBalanceShardsAllocator#allocate"; } }; - this.queue = new PendingListenersQueue(); + this.pendingListenersQueue = new PendingListenersQueue(); this.masterServiceTaskQueue = clusterService.createTaskQueue( "reconcile-desired-balance", Priority.URGENT, @@ -235,7 +255,7 @@ public void allocate(RoutingAllocation allocation, ActionListener listener var index = indexGenerator.incrementAndGet(); logger.debug("Executing allocate for [{}]", index); - queue.add(index, listener); + pendingListenersQueue.add(index, listener); // This can only run on master, so unset not-master if exists if (currentDesiredBalanceRef.compareAndSet(DesiredBalance.NOT_MASTER, DesiredBalance.BECOME_MASTER_INITIAL)) { logger.debug("initialized desired balance for becoming master"); @@ -378,7 +398,7 @@ public DesiredBalanceStats getStats() { private void onNoLongerMaster() { if (indexGenerator.getAndSet(-1) != -1) { currentDesiredBalanceRef.set(DesiredBalance.NOT_MASTER); - queue.completeAllAsNotMaster(); + pendingListenersQueue.completeAllAsNotMaster(); pendingDesiredBalanceMoves.clear(); desiredBalanceReconciler.clear(); desiredBalanceMetrics.zeroAllMetrics(); @@ -428,7 +448,7 @@ private ClusterState applyBalance( batchExecutionContext.initialState(), createReconcileAllocationAction(latest.getTask().desiredBalance) ); - latest.success(() -> queue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); + latest.success(() -> pendingListenersQueue.complete(latest.getTask().desiredBalance.lastConvergedIndex())); return newState; } } @@ -447,7 +467,7 @@ private static void discardSupersededTasks( // only for tests - in production, this happens after reconciliation protected final void completeToLastConvergedIndex() { - queue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); + pendingListenersQueue.complete(currentDesiredBalanceRef.get().lastConvergedIndex()); } private void recordTime(CounterMetric metric, Runnable action) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java index e1b58cf79ac09..5b14277f2c651 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/PendingListenersQueue.java @@ -24,6 +24,10 @@ import static org.elasticsearch.cluster.service.ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME; import static org.elasticsearch.cluster.service.MasterService.MASTER_UPDATE_THREAD_NAME; +/** + * Registers listeners with an `index` number ({@link #add(long, ActionListener)}) and then completes them whenever the latest index number + * is greater or equal to a listener's index value ({@link #complete(long)}). + */ public class PendingListenersQueue { private static final Logger logger = LogManager.getLogger(PendingListenersQueue.class); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java index 4fb9137cb4544..2bd1b9bb2bb64 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/ShardAssignment.java @@ -17,6 +17,14 @@ import static java.util.Collections.unmodifiableSet; +/** + * Simple shard assignment summary of shard copies for a particular index shard. + * + * @param nodeIds The node IDs of nodes holding a shard copy. + * @param total The total number of shard copies. + * @param unassigned The number of unassigned shard copies. + * @param ignored The number of ignored shard copies. + */ public record ShardAssignment(Set nodeIds, int total, int unassigned, int ignored) { public ShardAssignment { @@ -28,9 +36,13 @@ public int assigned() { return nodeIds.size(); } - public static ShardAssignment ofAssignedShards(List routings) { + /** + * Helper method to instantiate a new ShardAssignment from a given list of ShardRouting instances. Assumes all shards are assigned. + */ + public static ShardAssignment createFromAssignedShardRoutingsList(List routings) { var nodeIds = new LinkedHashSet(); for (ShardRouting routing : routings) { + assert routing.unassignedInfo() == null : "Expected assigned shard copies only, unassigned info: " + routing.unassignedInfo(); nodeIds.add(routing.currentNodeId()); } return new ShardAssignment(unmodifiableSet(nodeIds), routings.size(), 0, 0); diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index c34d0d19988c8..05d4b29f8f28f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -158,6 +158,31 @@ public void run() { } } + private record TimedListener(ActionListener listener, Recorder recorder) implements ActionListener { + + @Override + public void onResponse(Void response) { + try (Releasable ignored = recorder.record("listener.onResponse")) { + listener.onResponse(null); + } catch (Exception e) { + assert false : e; + logger.error("exception thrown by listener.onResponse", e); + } + } + + @Override + public void onFailure(Exception e) { + assert e != null; + try (Releasable ignored = recorder.record("listener.onFailure")) { + listener.onFailure(e); + } catch (Exception inner) { + e.addSuppressed(inner); + assert false : e; + logger.error(() -> "exception thrown by listener.onFailure", e); + } + } + } + @Override protected synchronized void doStop() { for (Map.Entry onGoingTimeout : timeoutClusterStateListeners.entrySet()) { @@ -394,12 +419,14 @@ private void runTask(String source, Function updateF final long startTimeMillis = threadPool.relativeTimeInMillis(); final Recorder stopWatch = new Recorder(threadPool, slowTaskThreadDumpTimeout); + final TimedListener timedListener = new TimedListener(clusterApplyListener, stopWatch); final ClusterState newClusterState; try { try (Releasable ignored = stopWatch.record("running task [" + source + ']')) { newClusterState = updateFunction.apply(previousClusterState); } } catch (Exception e) { + timedListener.onFailure(e); TimeValue executionTime = getTimeSince(startTimeMillis); logger.trace( () -> format( @@ -412,15 +439,14 @@ private void runTask(String source, Function updateF e ); warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch); - clusterApplyListener.onFailure(e); return; } if (previousClusterState == newClusterState) { + timedListener.onResponse(null); TimeValue executionTime = getTimeSince(startTimeMillis); logger.debug("processing [{}]: took [{}] no change in cluster state", source, executionTime); warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch); - clusterApplyListener.onResponse(null); } else { if (logger.isTraceEnabled()) { logger.debug("cluster state updated, version [{}], source [{}]\n{}", newClusterState.version(), source, newClusterState); @@ -430,6 +456,7 @@ private void runTask(String source, Function updateF try { setIsApplyingClusterState(); applyChanges(previousClusterState, newClusterState, source, stopWatch); + timedListener.onResponse(null); TimeValue executionTime = getTimeSince(startTimeMillis); logger.debug( "processing [{}]: took [{}] done applying updated cluster state (version: {}, uuid: {})", @@ -439,8 +466,11 @@ private void runTask(String source, Function updateF newClusterState.stateUUID() ); warnAboutSlowTaskIfNeeded(executionTime, source, stopWatch); - clusterApplyListener.onResponse(null); } catch (Exception e) { + // failing to apply a cluster state with an exception indicates a bug in validation or in one of the appliers; if we + // continue we will retry with the same cluster state but that might not help. + assert applicationMayFail(); + timedListener.onFailure(e); TimeValue executionTime = getTimeSince(startTimeMillis); if (logger.isTraceEnabled()) { logger.warn(() -> format(""" @@ -460,10 +490,6 @@ private void runTask(String source, Function updateF e ); } - // failing to apply a cluster state with an exception indicates a bug in validation or in one of the appliers; if we - // continue we will retry with the same cluster state but that might not help. - assert applicationMayFail(); - clusterApplyListener.onFailure(e); } finally { clearIsApplyingClusterState(); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index afee2491672ef..3c1f53ca4a2c9 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.store.FsDirectoryFactory; import org.elasticsearch.index.store.Store; @@ -191,7 +190,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING, IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING, + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index aec9c108d898d..16c6844f46402 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -1727,7 +1727,7 @@ public static > Setting enumSetting( * * @param key the key for the setting * @param defaultValue the default value for this setting - * @param properties properties properties for this setting like scope, filtering... + * @param properties properties for this setting like scope, filtering... * @return the setting object */ public static Setting memorySizeSetting(String key, ByteSizeValue defaultValue, Property... properties) { diff --git a/server/src/main/java/org/elasticsearch/health/HealthFeatures.java b/server/src/main/java/org/elasticsearch/health/HealthFeatures.java deleted file mode 100644 index 72fc955320b94..0000000000000 --- a/server/src/main/java/org/elasticsearch/health/HealthFeatures.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.health; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class HealthFeatures implements FeatureSpecification { - - public static final NodeFeature SUPPORTS_EXTENDED_REPOSITORY_INDICATOR = new NodeFeature("health.extended_repository_indicator", true); - - @Override - public Set getFeatures() { - return Set.of(SUPPORTS_EXTENDED_REPOSITORY_INDICATOR); - } -} diff --git a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java index aab9e972cba73..113e789727f0a 100644 --- a/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java +++ b/server/src/main/java/org/elasticsearch/health/node/LocalHealthMonitor.java @@ -24,7 +24,6 @@ import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.common.util.concurrent.RunOnce; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.health.metadata.HealthMetadata; import org.elasticsearch.health.node.action.HealthNodeNotDiscoveredException; import org.elasticsearch.health.node.selection.HealthNode; @@ -62,7 +61,6 @@ public class LocalHealthMonitor implements ClusterStateListener { private final ClusterService clusterService; private final ThreadPool threadPool; private final Client client; - private final FeatureService featureService; private volatile TimeValue monitorInterval; private volatile boolean enabled; @@ -88,7 +86,6 @@ private LocalHealthMonitor( ClusterService clusterService, ThreadPool threadPool, Client client, - FeatureService featureService, List> healthTrackers ) { this.threadPool = threadPool; @@ -96,7 +93,6 @@ private LocalHealthMonitor( this.enabled = HealthNodeTaskExecutor.ENABLED_SETTING.get(settings); this.clusterService = clusterService; this.client = client; - this.featureService = featureService; this.healthTrackers = healthTrackers; } @@ -105,17 +101,9 @@ public static LocalHealthMonitor create( ClusterService clusterService, ThreadPool threadPool, Client client, - FeatureService featureService, List> healthTrackers ) { - LocalHealthMonitor localHealthMonitor = new LocalHealthMonitor( - settings, - clusterService, - threadPool, - client, - featureService, - healthTrackers - ); + LocalHealthMonitor localHealthMonitor = new LocalHealthMonitor(settings, clusterService, threadPool, client, healthTrackers); localHealthMonitor.registerListeners(); return localHealthMonitor; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index f0814577bd203..7287a0bf307b9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -623,10 +623,7 @@ public Settings getAdditionalIndexSettings( } } if (indexMode == LOOKUP) { - return Settings.builder() - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all") - .build(); + return Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build(); } else { return Settings.EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 68f334b10ea52..284140460a437 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -52,7 +51,6 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; /** * This class encapsulates all index level settings and handles settings updates. @@ -655,48 +653,6 @@ public Iterator> settings() { Property.Final ); - public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( - "index.recovery.use_synthetic_source", - false, - new Setting.Validator<>() { - @Override - public void validate(Boolean value) {} - - @Override - public void validate(Boolean enabled, Map, Object> settings) { - if (enabled == false) { - return; - } - - // Verify if synthetic source is enabled on the index; fail if it is not - var indexMode = (IndexMode) settings.get(MODE); - if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { - var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); - if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", - RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), - INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), - SourceFieldMapper.Mode.SYNTHETIC.name(), - sourceMode.name() - ) - ); - } - } - } - - @Override - public Iterator> settings() { - List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); - return res.iterator(); - } - }, - Property.IndexScope, - Property.Final - ); - /** * Returns true if TSDB encoding is enabled. The default is true */ @@ -753,6 +709,60 @@ public Iterator> settings() { Property.ServerlessPublic ); + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting( + SourceFieldMapper.Mode.class, + settings -> { + final IndexMode indexMode = IndexSettings.MODE.get(settings); + return indexMode.defaultSourceMode().name(); + }, + "index.mapping.source.mode", + value -> {}, + Setting.Property.Final, + Setting.Property.IndexScope + ); + + public static final Setting RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting( + "index.recovery.use_synthetic_source", + false, + new Setting.Validator<>() { + @Override + public void validate(Boolean value) {} + + @Override + public void validate(Boolean enabled, Map, Object> settings) { + if (enabled == false) { + return; + } + + // Verify if synthetic source is enabled on the index; fail if it is not + var indexMode = (IndexMode) settings.get(MODE); + if (indexMode.defaultSourceMode() != SourceFieldMapper.Mode.SYNTHETIC) { + var sourceMode = (SourceFieldMapper.Mode) settings.get(INDEX_MAPPER_SOURCE_MODE_SETTING); + if (sourceMode != SourceFieldMapper.Mode.SYNTHETIC) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted when [%s] is set to [%s]. Current mode: [%s].", + RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), + INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + SourceFieldMapper.Mode.SYNTHETIC.name(), + sourceMode.name() + ) + ); + } + } + } + + @Override + public Iterator> settings() { + List> res = List.of(INDEX_MAPPER_SOURCE_MODE_SETTING, MODE); + return res.iterator(); + } + }, + Property.IndexScope, + Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -806,8 +816,6 @@ private static String getIgnoreAboveDefaultValue(final Settings settings) { } } - public static final NodeFeature IGNORE_ABOVE_INDEX_LEVEL_SETTING = new NodeFeature("mapper.ignore_above_index_level_setting", true); - private final Index index; private final IndexVersion version; private final Logger logger; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 9de463ec5f6f6..77de1654cf4ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -26,7 +26,6 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -62,8 +61,6 @@ public class BooleanFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "boolean"; - static final NodeFeature BOOLEAN_DIMENSION = new NodeFeature("mapper.boolean_dimension", true); - public static class Values { public static final BytesRef TRUE = new BytesRef("T"); public static final BytesRef FALSE = new BytesRef("F"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 068a9828809dd..cf0c355a22e65 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.logging.Loggers; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -22,8 +21,6 @@ import java.util.List; public class DocumentMapper { - static final NodeFeature INDEX_SORTING_ON_NESTED = new NodeFeature("mapper.index_sorting_on_nested", true); - private final String type; private final CompressedXContent mappingSource; private final MappingLookup mappingLookup; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index e03494dcb5926..bdb3d97d4c187 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -56,7 +56,6 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(context -> new IgnoredSourceFieldMapper(context.getIndexSettings())); - static final NodeFeature TRACK_IGNORED_SOURCE = new NodeFeature("mapper.track_ignored_source", true); static final NodeFeature DONT_EXPAND_DOTS_IN_IGNORED_SOURCE = new NodeFeature("mapper.ignored_source.dont_expand_dots"); static final NodeFeature IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD = new NodeFeature( "mapper.ignored_source_as_top_level_metadata_array_field" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java index e539c07caef61..9708753926e1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexModeFieldMapper.java @@ -13,7 +13,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -31,8 +30,6 @@ public class IndexModeFieldMapper extends MetadataFieldMapper { - static final NodeFeature QUERYING_INDEX_MODE = new NodeFeature("mapper.query_index_mode", true); - public static final String NAME = "_index_mode"; public static final String CONTENT_TYPE = "_index_mode"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index a26a4bb80d50e..bdcf9bf98279f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -89,9 +88,6 @@ public final class KeywordFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "keyword"; - static final NodeFeature KEYWORD_DIMENSION_IGNORE_ABOVE = new NodeFeature("mapper.keyword_dimension_ignore_above", true); - static final NodeFeature KEYWORD_NORMALIZER_SYNTHETIC_SOURCE = new NodeFeature("mapper.keyword_normalizer_synthetic_source", true); - public static class Defaults { public static final FieldType FIELD_TYPE; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index f293ced122d23..bafa74b662f00 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.util.StringLiteralDeduplicator; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -30,8 +29,6 @@ public abstract class Mapper implements ToXContentFragment, Iterable { - public static final NodeFeature SYNTHETIC_SOURCE_KEEP_FEATURE = new NodeFeature("mapper.synthetic_source_keep", true); - public static final String SYNTHETIC_SOURCE_KEEP_PARAM = "synthetic_source_keep"; // Only relevant for synthetic source mode. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 5dbaf0e0f40ad..8e669a91fd9ea 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -11,9 +11,6 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; -import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.util.Set; @@ -28,33 +25,7 @@ public class MapperFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of( - BWC_WORKAROUND_9_0, - IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, - PassThroughObjectMapper.PASS_THROUGH_PRIORITY, - RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, - SourceFieldMapper.SYNTHETIC_SOURCE_FALLBACK, - DenseVectorFieldMapper.INT4_QUANTIZATION, - DenseVectorFieldMapper.BIT_VECTORS, - DocumentMapper.INDEX_SORTING_ON_NESTED, - KeywordFieldMapper.KEYWORD_DIMENSION_IGNORE_ABOVE, - IndexModeFieldMapper.QUERYING_INDEX_MODE, - NodeMappingStats.SEGMENT_LEVEL_FIELDS_STATS, - BooleanFieldMapper.BOOLEAN_DIMENSION, - ObjectMapper.SUBOBJECTS_AUTO, - ObjectMapper.SUBOBJECTS_AUTO_FIXES, - KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE, - SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, - Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, - SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT, - SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_FIX, - FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT, - IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING, - SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX, - TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF, - FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT, - DenseVectorFieldMapper.BBQ_FORMAT - ); + return Set.of(BWC_WORKAROUND_9_0); } public static final NodeFeature CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX = new NodeFeature( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java index e7ca7367832b6..0987c6dfb8c8b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NodeMappingStats.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -28,8 +27,6 @@ */ public class NodeMappingStats implements Writeable, ToXContentFragment { - public static final NodeFeature SEGMENT_LEVEL_FIELDS_STATS = new NodeFeature("mapper.segment_level_fields_stats", true); - private static final class Fields { static final String MAPPINGS = "mappings"; static final String TOTAL_COUNT = "total_count"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index e734a8e5b4377..86ce4fbb74837 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -52,9 +51,6 @@ public class ObjectMapper extends Mapper { public static final String CONTENT_TYPE = "object"; static final String STORE_ARRAY_SOURCE_PARAM = "store_array_source"; - static final NodeFeature SUBOBJECTS_AUTO = new NodeFeature("mapper.subobjects_auto", true); - // No-op. All uses of this feature were reverted but node features can't be removed. - static final NodeFeature SUBOBJECTS_AUTO_FIXES = new NodeFeature("mapper.subobjects_auto_fixes", true); /** * Enhances the previously boolean option for subobjects support with an intermediate mode `auto` that uses diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index d16acab11a508..fbf8dd4538037 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -10,7 +10,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Explicit; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.xcontent.XContentBuilder; @@ -39,8 +38,6 @@ public class PassThroughObjectMapper extends ObjectMapper { public static final String CONTENT_TYPE = "passthrough"; public static final String PRIORITY_PARAM_NAME = "priority"; - static final NodeFeature PASS_THROUGH_PRIORITY = new NodeFeature("mapper.pass_through_priority", true); - public static class Builder extends ObjectMapper.Builder { // Controls whether subfields are configured as time-series dimensions. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 36f61311ddfc7..461ad74a9434d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -51,7 +51,6 @@ /** A {@link FieldMapper} for indexing numeric and date ranges, and creating queries */ public class RangeFieldMapper extends FieldMapper { - public static final NodeFeature NULL_VALUES_OFF_BY_ONE_FIX = new NodeFeature("mapper.range.null_values_off_by_one_fix", true); public static final NodeFeature DATE_RANGE_INDEXING_FIX = new NodeFeature("mapper.range.date_range_indexing_fix"); public static final boolean DEFAULT_INCLUDE_UPPER = true; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 29acdf200692d..44c0193206efc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -18,7 +18,6 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.core.Nullable; @@ -41,20 +40,6 @@ import java.util.Locale; public class SourceFieldMapper extends MetadataFieldMapper { - public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback", true); - public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature( - "mapper.source.synthetic_source_stored_fields_advance_fix", - true - ); - public static final NodeFeature SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT = new NodeFeature( - "mapper.source.synthetic_source_with_copy_to_and_doc_values_false", - true - ); - public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_FIX = new NodeFeature("mapper.source.synthetic_source_copy_to_fix", true); - public static final NodeFeature SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX = new NodeFeature( - "mapper.source.synthetic_source_copy_to_inside_objects_fix", - true - ); public static final NodeFeature REMOVE_SYNTHETIC_SOURCE_ONLY_VALIDATION = new NodeFeature( "mapper.source.remove_synthetic_source_only_validation" ); @@ -70,11 +55,6 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final String LOSSY_PARAMETERS_ALLOWED_SETTING_NAME = "index.lossy.source-mapping-parameters"; - public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(SourceFieldMapper.Mode.class, settings -> { - final IndexMode indexMode = IndexSettings.MODE.get(settings); - return indexMode.defaultSourceMode().name(); - }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope); - public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed " + "in future versions. Use [index.mapping.source.mode] index setting instead."; @@ -264,8 +244,8 @@ public SourceFieldMapper build() { private Mode resolveSourceMode() { // If the `index.mapping.source.mode` exists it takes precedence to determine the source mode for `_source` // otherwise the mode is determined according to `_source.mode`. - if (INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(settings)) { + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings); } // If `_source.mode` is not set we need to apply a default according to index mode. @@ -297,7 +277,7 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) { return DEFAULT; } - final Mode settingSourceMode = INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); + final Mode settingSourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(c.getSettings()); // Needed for bwc so that "mode" is not serialized in case of standard index with stored source. if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) { return DEFAULT; @@ -482,11 +462,11 @@ public boolean isSynthetic() { } public static boolean isSynthetic(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC; } public static boolean isStored(IndexSettings indexSettings) { - return INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; + return IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexSettings.getSettings()) == Mode.STORED; } public boolean isDisabled() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java index 2a7069c5a52e3..4a6ba5d1fa800 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.fielddata.FieldData; @@ -47,10 +46,6 @@ public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper { public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper(); public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper()); - static final NodeFeature TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF = new NodeFeature( - "tsdb.ts_routing_hash_doc_value_parse_byte_ref", - true - ); public static final DocValueFormat TS_ROUTING_HASH_DOC_VALUE_FORMAT = TimeSeriesRoutingHashFieldType.DOC_VALUE_FORMAT; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index fc3f297f97252..7ef12f6dd30d2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -112,12 +111,6 @@ */ public final class FlattenedFieldMapper extends FieldMapper { - public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support", true); - public static final NodeFeature IGNORE_ABOVE_WITH_ARRAYS_SUPPORT = new NodeFeature( - "mapper.flattened.ignore_above_with_arrays_support", - true - ); - public static final String CONTENT_TYPE = "flattened"; public static final String KEYED_FIELD_SUFFIX = "._keyed"; public static final String KEYED_IGNORED_VALUES_FIELD_SUFFIX = "._keyed._ignored"; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 3e0656205b976..5edff48577efc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -38,7 +38,6 @@ import org.apache.lucene.util.VectorUtil; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat; @@ -108,10 +107,6 @@ public static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; } - public static final NodeFeature INT4_QUANTIZATION = new NodeFeature("mapper.vectors.int4_quantization", true); - public static final NodeFeature BIT_VECTORS = new NodeFeature("mapper.vectors.bit_vectors", true); - public static final NodeFeature BBQ_FORMAT = new NodeFeature("mapper.vectors.bbq", true); - public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0; public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION; public static final IndexVersion NORMALIZE_COSINE = IndexVersions.NORMALIZED_VECTOR_COSINE; diff --git a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java index 5bf5182af336e..56c08f2b2fb85 100644 --- a/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java +++ b/server/src/main/java/org/elasticsearch/ingest/EnterpriseGeoIpTask.java @@ -13,7 +13,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.persistent.PersistentTaskParams; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.XContentBuilder; @@ -36,10 +35,6 @@ private EnterpriseGeoIpTask() { } public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise-geoip-downloader"; - public static final NodeFeature GEOIP_DOWNLOADER_DATABASE_CONFIGURATION = new NodeFeature( - "geoip.downloader.database.configuration", - true - ); public static class EnterpriseGeoIpTaskParams implements PersistentTaskParams { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java b/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java deleted file mode 100644 index 7c12b180b4607..0000000000000 --- a/server/src/main/java/org/elasticsearch/ingest/IngestGeoIpFeatures.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.ingest; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -import static org.elasticsearch.ingest.EnterpriseGeoIpTask.GEOIP_DOWNLOADER_DATABASE_CONFIGURATION; - -public class IngestGeoIpFeatures implements FeatureSpecification { - - public static final NodeFeature GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE = new NodeFeature( - "get_database_configuration_action.multi_node", - true - ); - - public static final NodeFeature PUT_DATABASE_CONFIGURATION_ACTION_IPINFO = new NodeFeature( - "put_database_configuration_action.ipinfo", - true - ); - - public Set getFeatures() { - return Set.of( - GEOIP_DOWNLOADER_DATABASE_CONFIGURATION, - GET_DATABASE_CONFIGURATION_ACTION_MULTI_NODE, - PUT_DATABASE_CONFIGURATION_ACTION_IPINFO - ); - } -} diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index 5e9f7bb99b31c..4693b4fcf718a 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -1200,7 +1200,6 @@ public Map searchFields() { DataStreamAutoShardingService dataStreamAutoShardingService = new DataStreamAutoShardingService( settings, clusterService, - featureService, threadPool::absoluteTimeInMillis ); dataStreamAutoShardingService.init(); @@ -1351,7 +1350,7 @@ private Module loadDiagnosticServices( var serverHealthIndicatorServices = Stream.of( new StableMasterHealthIndicatorService(coordinationDiagnosticsService, clusterService), - new RepositoryIntegrityHealthIndicatorService(clusterService, featureService), + new RepositoryIntegrityHealthIndicatorService(clusterService), new DiskHealthIndicatorService(clusterService, featureService), new ShardsCapacityHealthIndicatorService(clusterService, featureService), fileSettingsHealthIndicatorService @@ -1376,14 +1375,7 @@ private Module loadDiagnosticServices( new DiskHealthTracker(nodeService, clusterService), new RepositoriesHealthTracker(repositoriesService) ); - LocalHealthMonitor localHealthMonitor = LocalHealthMonitor.create( - settings, - clusterService, - threadPool, - client, - featureService, - healthTrackers - ); + LocalHealthMonitor localHealthMonitor = LocalHealthMonitor.create(settings, clusterService, threadPool, client, healthTrackers); HealthInfoCache nodeHealthOverview = HealthInfoCache.create(clusterService); return b -> { diff --git a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java index de56ead9b5aba..1a169699d4131 100644 --- a/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java +++ b/server/src/main/java/org/elasticsearch/readiness/ReadinessService.java @@ -22,9 +22,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.env.Environment; -import org.elasticsearch.reservedstate.service.FileSettingsFeatures; import org.elasticsearch.reservedstate.service.FileSettingsService; import org.elasticsearch.shutdown.PluginShutdownService; import org.elasticsearch.transport.BindTransportException; @@ -280,22 +278,7 @@ private boolean isMasterElected(ClusterState clusterState) { // protected to allow mock service to override protected boolean areFileSettingsApplied(ClusterState clusterState) { ReservedStateMetadata fileSettingsMetadata = clusterState.metadata().reservedStateMetadata().get(FileSettingsService.NAMESPACE); - if (fileSettingsMetadata == null) { - // In order to block readiness on file settings being applied, we need to know that the master node has written an initial - // version, or a marker that file settings don't exist. When upgrading from a version that did not have file settings, the - // current master node may not be the first node upgraded. To be safe, we wait to consider file settings application for - // readiness until the whole cluster supports file settings. Note that this only applies when no reserved state metadata - // exists, so either we are starting up a current cluster (and the feature will be found) or we are upgrading from - // a version before file settings existed (before 8.4). - return supportsFileSettings(clusterState) == false; - } else { - return fileSettingsMetadata.version().equals(ReservedStateMetadata.NO_VERSION) == false; - } - } - - @SuppressForbidden(reason = "need to check file settings support on exact cluster state") - private boolean supportsFileSettings(ClusterState clusterState) { - return clusterState.clusterFeatures().clusterHasFeature(clusterState.nodes(), FileSettingsFeatures.FILE_SETTINGS_SUPPORTED); + return fileSettingsMetadata != null && fileSettingsMetadata.version().equals(ReservedStateMetadata.NO_VERSION) == false; } private void setReady(boolean ready) { diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java deleted file mode 100644 index b6dea6a2003fc..0000000000000 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesFeatures.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.repositories; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class RepositoriesFeatures implements FeatureSpecification { - public static final NodeFeature SUPPORTS_REPOSITORIES_USAGE_STATS = new NodeFeature("repositories.supports_usage_stats", true); - - @Override - public Set getFeatures() { - return Set.of(SUPPORTS_REPOSITORIES_USAGE_STATS); - } -} diff --git a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java b/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java deleted file mode 100644 index a60f525be988a..0000000000000 --- a/server/src/main/java/org/elasticsearch/reservedstate/service/FileSettingsFeatures.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.reservedstate.service; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; - -import java.util.Set; - -public class FileSettingsFeatures implements FeatureSpecification { - - // Although file settings were supported starting in 8.4.0, this is really about whether file settings - // are used in readiness. - public static final NodeFeature FILE_SETTINGS_SUPPORTED = new NodeFeature("file_settings", true); - - @Override - public Set getFeatures() { - return Set.of(FILE_SETTINGS_SUPPORTED); - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java index 07cc73f4da2b1..31758be719a66 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java +++ b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.xcontent.MediaType; import org.elasticsearch.xcontent.ParsedMediaType; @@ -27,7 +26,6 @@ class RestCompatibleVersionHelper { /** * @return The requested API version, or {@link Optional#empty()} if there was no explicit version in the request. */ - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) static Optional getCompatibleVersion( @Nullable ParsedMediaType acceptHeader, @Nullable ParsedMediaType contentTypeHeader, @@ -52,8 +50,7 @@ static Optional getCompatibleVersion( if (hasContent) { // content-type version must be current or prior - // This can be uncommented once all references to RestApiVersion.V_7 are removed - /*if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) { + if (contentTypeVersion > RestApiVersion.current().major || contentTypeVersion < RestApiVersion.minimumSupported().major) { throw new ElasticsearchStatusException( "Content-Type version must be either version {} or {}, but found {}. Content-Type={}", RestStatus.BAD_REQUEST, @@ -62,7 +59,7 @@ static Optional getCompatibleVersion( contentTypeVersion, contentTypeHeader ); - }*/ + } // if both accept and content-type are sent, the version must match if (contentTypeVersion != acceptVersion) { throw new ElasticsearchStatusException( diff --git a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java b/server/src/main/java/org/elasticsearch/rest/RestFeatures.java deleted file mode 100644 index e72b30526c8e3..0000000000000 --- a/server/src/main/java/org/elasticsearch/rest/RestFeatures.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.rest; - -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction; - -import java.util.Set; - -import static org.elasticsearch.search.fetch.subphase.highlight.DefaultHighlighter.UNIFIED_HIGHLIGHTER_MATCHED_FIELDS; - -public class RestFeatures implements FeatureSpecification { - @Override - public Set getFeatures() { - return Set.of( - RestNodesCapabilitiesAction.CAPABILITIES_ACTION, - RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES, - UNIFIED_HIGHLIGHTER_MATCHED_FIELDS - ); - } -} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java index ca9e4abcaeec7..477cc1acb8319 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java @@ -11,15 +11,16 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsAction; import org.elasticsearch.action.admin.cluster.settings.RestClusterGetSettingsResponse; -import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -52,19 +53,14 @@ public String getName() { return "cluster_get_settings_action"; } - private static void setUpRequestParams(MasterNodeReadRequest clusterRequest, RestRequest request) { - clusterRequest.local(request.paramAsBoolean("local", clusterRequest.local())); - } - @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final boolean renderDefaults = request.paramAsBoolean("include_defaults", false); ClusterGetSettingsAction.Request clusterSettingsRequest = new ClusterGetSettingsAction.Request(getMasterNodeTimeout(request)); + RestUtils.consumeDeprecatedLocalParameter(request); - setUpRequestParams(clusterSettingsRequest, request); - - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( ClusterGetSettingsAction.INSTANCE, clusterSettingsRequest, new RestToXContentListener(channel).map( diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java index 265cdd5979adf..7d660e527a814 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesCapabilitiesAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.admin.cluster.node.capabilities.NodesCapabilitiesRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; @@ -30,8 +29,6 @@ @ServerlessScope(Scope.INTERNAL) public class RestNodesCapabilitiesAction extends BaseRestHandler { - public static final NodeFeature CAPABILITIES_ACTION = new NodeFeature("rest.capabilities_action", true); - public static final NodeFeature LOCAL_ONLY_CAPABILITIES = new NodeFeature("rest.local_only_capabilities", true); private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of( "timeout", "method", diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java index ec8bb6285bdd4..da7a7d3379ee0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestUpdateDesiredNodesAction.java @@ -14,16 +14,13 @@ import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.metadata.DesiredNode; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; import java.util.List; -import java.util.function.Predicate; import static org.elasticsearch.rest.RestUtils.getAckTimeout; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @@ -33,11 +30,6 @@ public class RestUpdateDesiredNodesAction extends BaseRestHandler { private final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(RestUpdateDesiredNodesAction.class); private static final String VERSION_DEPRECATION_MESSAGE = "[version removal] Specifying node_version in desired nodes requests is deprecated."; - private final Predicate clusterSupportsFeature; - - public RestUpdateDesiredNodesAction(Predicate clusterSupportsFeature) { - this.clusterSupportsFeature = clusterSupportsFeature; - } @Override public String getName() { @@ -67,14 +59,8 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli ); } - if (clusterSupportsFeature.test(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED)) { - if (updateDesiredNodesRequest.getNodes().stream().anyMatch(DesiredNode::hasVersion)) { - deprecationLogger.compatibleCritical("desired_nodes_version", VERSION_DEPRECATION_MESSAGE); - } - } else { - if (updateDesiredNodesRequest.getNodes().stream().anyMatch(n -> n.hasVersion() == false)) { - throw new XContentParseException("[node_version] field is required and must have a valid value"); - } + if (updateDesiredNodesRequest.getNodes().stream().anyMatch(DesiredNode::hasVersion)) { + deprecationLogger.compatibleCritical("desired_nodes_version", VERSION_DEPRECATION_MESSAGE); } return restChannel -> client.execute( diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java index 27620fa750ea9..f9e02646041a6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetMappingAction.java @@ -13,7 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.http.HttpChannel; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -51,11 +50,9 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(getMasterNodeTimeout(request)); getMappingsRequest.indices(indices); getMappingsRequest.indicesOptions(IndicesOptions.fromRequest(request, getMappingsRequest.indicesOptions())); - final TimeValue timeout = getMasterNodeTimeout(request); - getMappingsRequest.masterNodeTimeout(timeout); getMappingsRequest.local(request.paramAsBoolean("local", getMappingsRequest.local())); final HttpChannel httpChannel = request.getHttpChannel(); return channel -> new RestCancellableNodeClient(client, httpChannel).admin() diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java index 4cc010c15ffb9..1689d234387cc 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateIndexTemplateAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -41,8 +42,10 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest(request.param("name")); - simulateIndexTemplateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); + SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest( + getMasterNodeTimeout(request), + request.param("name") + ); simulateIndexTemplateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -57,7 +60,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC simulateIndexTemplateRequest.indexTemplateRequest(indexTemplateRequest); } - return channel -> client.execute( + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( SimulateIndexTemplateAction.INSTANCE, simulateIndexTemplateRequest, new RestToXContentListener<>(channel) diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java index cc2e3136a4163..47b68e24a9ed0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSimulateTemplateAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; @@ -39,8 +40,10 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(); - simulateRequest.templateName(request.param("name")); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request( + getMasterNodeTimeout(request), + request.param("name") + ); simulateRequest.includeDefaults(request.paramAsBoolean("include_defaults", false)); if (request.hasContent()) { TransportPutComposableIndexTemplateAction.Request indexTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( @@ -54,8 +57,11 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli simulateRequest.indexTemplateRequest(indexTemplateRequest); } - simulateRequest.masterNodeTimeout(getMasterNodeTimeout(request)); - return channel -> client.execute(SimulateTemplateAction.INSTANCE, simulateRequest, new RestToXContentListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).execute( + SimulateTemplateAction.INSTANCE, + simulateRequest, + new RestToXContentListener<>(channel) + ); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java index 88756ddbf4ed5..cd781e42379d1 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java @@ -17,6 +17,6 @@ public final class ScriptFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(VectorScoreScriptUtils.HAMMING_DISTANCE_FUNCTION, ScriptTermStats.TERM_STAT_FEATURE); + return Set.of(); } } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java index 82f6e972e1266..9c51afce3a49d 100644 --- a/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java +++ b/server/src/main/java/org/elasticsearch/script/ScriptTermStats.java @@ -16,7 +16,6 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.util.CachedSupplier; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.search.internal.ContextIndexSearcher; import java.io.IOException; @@ -30,8 +29,6 @@ */ public class ScriptTermStats { - public static final NodeFeature TERM_STAT_FEATURE = new NodeFeature("script.term_stats", true); - private final IntSupplier docIdSupplier; private final Term[] terms; private final IndexSearcher searcher; diff --git a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java index 9b4d105eea100..bdebdcc1eecb4 100644 --- a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java +++ b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java @@ -10,7 +10,6 @@ package org.elasticsearch.script; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.DenseVectorDocValuesField; @@ -21,8 +20,6 @@ public class VectorScoreScriptUtils { - public static final NodeFeature HAMMING_DISTANCE_FUNCTION = new NodeFeature("script.hamming", true); - public static class DenseVectorFunction { protected final ScoreScript scoreScript; protected final DenseVectorDocValuesField field; diff --git a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java index 553511346b182..98dd7f9388c1f 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/SearchFeatures.java @@ -11,7 +11,6 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.search.vectors.KnnVectorQueryBuilder; import java.util.Set; @@ -21,7 +20,7 @@ public final class SearchFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of(KnnVectorQueryBuilder.K_PARAM_SUPPORTED, LUCENE_10_0_0_UPGRADE); + return Set.of(LUCENE_10_0_0_UPGRADE); } public static final NodeFeature RETRIEVER_RESCORER_ENABLED = new NodeFeature("search.retriever.rescorer.enabled"); diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 8c21abe4180ea..6d47493e4d063 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1402,9 +1402,6 @@ private SearchSourceBuilder parseXContent( } } else if (token == XContentParser.Token.START_OBJECT) { if (RETRIEVER.match(currentFieldName, parser.getDeprecationHandler())) { - if (clusterSupportsFeature.test(RetrieverBuilder.RETRIEVERS_SUPPORTED) == false) { - throw new ParsingException(parser.getTokenLocation(), "Unknown key for a START_OBJECT in [retriever]."); - } retrieverBuilder = RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, new RetrieverParserContext(searchUsage, clusterSupportsFeature) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java index 5220dadec7a1a..53116ff8f6ebf 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/MatchedQueriesPhase.java @@ -9,10 +9,12 @@ package org.elasticsearch.search.fetch.subphase; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.ScorerSupplier; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.search.fetch.FetchContext; @@ -52,8 +54,9 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) throws IOExcept ); } return new FetchSubPhaseProcessor() { + record ScorerAndIterator(Scorer scorer, DocIdSetIterator approximation, TwoPhaseIterator twoPhase) {} - final Map matchingIterators = new HashMap<>(); + final Map matchingIterators = new HashMap<>(); @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { @@ -63,7 +66,14 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { if (ss != null) { Scorer scorer = ss.get(0L); if (scorer != null) { - matchingIterators.put(entry.getKey(), scorer); + final TwoPhaseIterator twoPhase = scorer.twoPhaseIterator(); + final DocIdSetIterator iterator; + if (twoPhase == null) { + iterator = scorer.iterator(); + } else { + iterator = twoPhase.approximation(); + } + matchingIterators.put(entry.getKey(), new ScorerAndIterator(scorer, iterator, twoPhase)); } } } @@ -73,13 +83,13 @@ public void setNextReader(LeafReaderContext readerContext) throws IOException { public void process(HitContext hitContext) throws IOException { Map matches = new LinkedHashMap<>(); int doc = hitContext.docId(); - for (Map.Entry entry : matchingIterators.entrySet()) { - Scorer scorer = entry.getValue(); - if (scorer.iterator().docID() < doc) { - scorer.iterator().advance(doc); + for (Map.Entry entry : matchingIterators.entrySet()) { + ScorerAndIterator query = entry.getValue(); + if (query.approximation.docID() < doc) { + query.approximation.advance(doc); } - if (scorer.iterator().docID() == doc) { - matches.put(entry.getKey(), scorer.score()); + if (query.approximation.docID() == doc && (query.twoPhase == null || query.twoPhase.matches())) { + matches.put(entry.getKey(), query.scorer.score()); } } hitContext.hit().matchedQueries(matches); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java index b16617e2eb4d9..c47f815c18639 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/DefaultHighlighter.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.text.Text; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -51,8 +50,6 @@ public class DefaultHighlighter implements Highlighter { - public static final NodeFeature UNIFIED_HIGHLIGHTER_MATCHED_FIELDS = new NodeFeature("unified_highlighter_matched_fields", true); - @Override public boolean canHighlight(MappedFieldType fieldType) { return true; diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 9eb0170af5efb..d899a390d8be7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -102,14 +102,6 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.fieldLookupProvider = searchLookup.fieldLookupProvider; } - private SearchLookup(SearchLookup searchLookup, SourceProvider sourceProvider, Set fieldChain) { - this.fieldChain = Collections.unmodifiableSet(fieldChain); - this.sourceProvider = sourceProvider; - this.fieldTypeLookup = searchLookup.fieldTypeLookup; - this.fieldDataLookup = searchLookup.fieldDataLookup; - this.fieldLookupProvider = searchLookup.fieldLookupProvider; - } - /** * Creates a copy of the current {@link SearchLookup} that looks fields up in the same way, but also tracks field references * in order to detect cycles and prevent resolving fields that depend on more than {@link #MAX_FIELD_CHAIN_DEPTH} other fields. @@ -153,7 +145,4 @@ public Source getSource(LeafReaderContext ctx, int doc) throws IOException { return sourceProvider.getSource(ctx, doc); } - public SearchLookup swapSourceProvider(SourceProvider sourceProvider) { - return new SearchLookup(this, sourceProvider, fieldChain); - } } diff --git a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java index e64bbe3c39d79..4374c06da365d 100644 --- a/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java +++ b/server/src/main/java/org/elasticsearch/search/rank/feature/RankFeatureShardPhase.java @@ -83,7 +83,7 @@ public static void processFetch(SearchContext searchContext) { // FetchSearchResult#shardResult() SearchHits hits = fetchSearchResult.hits(); RankFeatureShardResult featureRankShardResult = (RankFeatureShardResult) rankFeaturePhaseRankShardContext - .buildRankFeatureShardResult(hits, searchContext.shardTarget().getShardId().id()); + .buildRankFeatureShardResult(hits, searchContext.request().shardRequestIndex()); // save the result in the search context // need to add profiling info as well available from fetch if (featureRankShardResult != null) { diff --git a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java index ecc03d05b28a6..737d2aa397c34 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/KnnRetrieverBuilder.java @@ -10,8 +10,6 @@ package org.elasticsearch.search.retriever; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -46,7 +44,6 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { public static final String NAME = "knn"; - public static final NodeFeature KNN_RETRIEVER_SUPPORTED = new NodeFeature("knn_retriever_supported", true); public static final ParseField FIELD_FIELD = new ParseField("field"); public static final ParseField K_FIELD = new ParseField("k"); @@ -103,9 +100,6 @@ public final class KnnRetrieverBuilder extends RetrieverBuilder { } public static KnnRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException { - if (context.clusterSupportsFeature(KNN_RETRIEVER_SUPPORTED) == false) { - throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]"); - } return PARSER.apply(parser, context); } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java index 357555cc59942..ce852a44c28ec 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieverBuilder.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.SuggestingErrorOnUnknown; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -50,8 +49,6 @@ */ public abstract class RetrieverBuilder implements Rewriteable, ToXContent { - public static final NodeFeature RETRIEVERS_SUPPORTED = new NodeFeature("retrievers_supported", true); - public static final ParseField PRE_FILTER_FIELD = new ParseField("filter"); public static final ParseField MIN_SCORE_FIELD = new ParseField("min_score"); diff --git a/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java b/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java index 74a8b30c8e7dc..bfd6f572a9e65 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/RetrieversFeatures.java @@ -22,10 +22,6 @@ public class RetrieversFeatures implements FeatureSpecification { @Override public Set getFeatures() { - return Set.of( - RetrieverBuilder.RETRIEVERS_SUPPORTED, - StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED, - KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED - ); + return Set.of(); } } diff --git a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java index 2ffb9e3a98028..3ca74dc133d47 100644 --- a/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/retriever/StandardRetrieverBuilder.java @@ -9,8 +9,6 @@ package org.elasticsearch.search.retriever; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -39,7 +37,6 @@ public final class StandardRetrieverBuilder extends RetrieverBuilder implements ToXContent { public static final String NAME = "standard"; - public static final NodeFeature STANDARD_RETRIEVER_SUPPORTED = new NodeFeature("standard_retriever_supported", true); public static final ParseField QUERY_FIELD = new ParseField("query"); public static final ParseField SEARCH_AFTER_FIELD = new ParseField("search_after"); @@ -81,9 +78,6 @@ public final class StandardRetrieverBuilder extends RetrieverBuilder implements } public static StandardRetrieverBuilder fromXContent(XContentParser parser, RetrieverParserContext context) throws IOException { - if (context.clusterSupportsFeature(STANDARD_RETRIEVER_SUPPORTED) == false) { - throw new ParsingException(parser.getTokenLocation(), "unknown retriever [" + NAME + "]"); - } return PARSER.apply(parser, context); } diff --git a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java index 193191658af08..565fd7325a5ac 100644 --- a/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/vectors/KnnVectorQueryBuilder.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.core.Nullable; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedObjectMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -56,8 +55,6 @@ * {@link org.apache.lucene.search.KnnByteVectorQuery}. */ public class KnnVectorQueryBuilder extends AbstractQueryBuilder { - public static final NodeFeature K_PARAM_SUPPORTED = new NodeFeature("search.vectors.k_param_supported", true); - public static final String NAME = "knn"; private static final int NUM_CANDS_LIMIT = 10_000; private static final float NUM_CANDS_MULTIPLICATIVE_FACTOR = 1.5f; diff --git a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java index 4be1ee9ddc513..a6a7b458c216d 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorService.java @@ -13,9 +13,7 @@ import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.health.Diagnosis; -import org.elasticsearch.health.HealthFeatures; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorImpact; import org.elasticsearch.health.HealthIndicatorResult; @@ -100,11 +98,9 @@ public class RepositoryIntegrityHealthIndicatorService implements HealthIndicato ); private final ClusterService clusterService; - private final FeatureService featureService; - public RepositoryIntegrityHealthIndicatorService(ClusterService clusterService, FeatureService featureService) { + public RepositoryIntegrityHealthIndicatorService(ClusterService clusterService) { this.clusterService = clusterService; - this.featureService = featureService; } @Override @@ -175,15 +171,8 @@ private RepositoryHealthAnalyzer( || invalidRepositories.isEmpty() == false) { healthStatus = YELLOW; } else if (repositoriesHealthByNode.isEmpty()) { - clusterHasFeature = featureService.clusterHasFeature( - clusterState, - HealthFeatures.SUPPORTS_EXTENDED_REPOSITORY_INDICATOR - ) == false; - if (clusterHasFeature) { - healthStatus = GREEN; - } else { - healthStatus = UNKNOWN; - } + clusterHasFeature = false; + healthStatus = UNKNOWN; } else { healthStatus = GREEN; } diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index f260c7aad30e5..debe3d6e6bd92 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -70,7 +70,6 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.Mapping; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.shard.IndexLongFieldRange; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; @@ -158,7 +157,7 @@ public final class RestoreService implements ClusterStateApplier { SETTING_CREATION_DATE, SETTING_HISTORY_UUID, IndexSettings.MODE.getKey(), - SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), + IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), IndexSortConfig.INDEX_SORT_ORDER_SETTING.getKey(), diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index f55e3740aaa8f..37a3ec586d104 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.node.Node; import org.elasticsearch.node.ReportingService; import org.elasticsearch.telemetry.metric.Instrument; @@ -120,13 +119,7 @@ public static class Names { public static final String THREAD_POOL_METRIC_NAME_REJECTED = ".threads.rejected.total"; public enum ThreadPoolType { - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9 - DIRECT("direct"), FIXED("fixed"), - @Deprecated(forRemoval = true) - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // no longer used, remove in v9 - FIXED_AUTO_QUEUE_SIZE("fixed_auto_queue_size"), SCALING("scaling"); private final String type; diff --git a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java index a43b1e7440bda..ec9b30e71e62c 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportHandshaker.java @@ -41,9 +41,14 @@ final class TransportHandshaker { * ignores the body of the request. After the handshake, the OutboundHandler uses the min(local,remote) protocol version for all later * messages. * - * This version supports two handshake protocols, v6080099 and v7170099, which respectively have the same message structure as the - * transport protocols of v6.8.0 and v7.17.0. This node only sends v7170099 requests, but it can send a valid response to any v6080099 - * requests that it receives. + * This version supports three handshake protocols, v6080099, v7170099 and v8800000, which respectively have the same message structure + * as the transport protocols of v6.8.0, v7.17.0, and v8.18.0. This node only sends v7170099 requests, but it can send a valid response + * to any v6080099 or v8800000 requests that it receives. + * + * Note that these are not really TransportVersion constants as used elsewhere in ES, they're independent things that just happen to be + * stored in the same location in the message header and which roughly match the same ID numbering scheme. Older versions of ES did + * rely on them matching the real transport protocol (which itself matched the release version numbers), but these days that's no longer + * true. * * Here are some example messages, broken down to show their structure: * @@ -79,7 +84,7 @@ final class TransportHandshaker { * c3 f9 eb 03 -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099) * * - * ## v7170099 Request: + * ## v7170099 and v8800000 Requests: * * 45 53 -- 'ES' marker * 00 00 00 31 -- total message length @@ -98,7 +103,7 @@ final class TransportHandshaker { * 04 -- payload length * c3 f9 eb 03 -- max acceptable protocol version (vInt: 00000011 11101011 11111001 11000011 == 8060099) * - * ## v7170099 Response: + * ## v7170099 and v8800000 Responses: * * 45 53 -- 'ES' marker * 00 00 00 17 -- total message length @@ -118,7 +123,12 @@ final class TransportHandshaker { static final TransportVersion EARLIEST_HANDSHAKE_VERSION = TransportVersion.fromId(6080099); static final TransportVersion REQUEST_HANDSHAKE_VERSION = TransportVersions.MINIMUM_COMPATIBLE; - static final Set ALLOWED_HANDSHAKE_VERSIONS = Set.of(EARLIEST_HANDSHAKE_VERSION, REQUEST_HANDSHAKE_VERSION); + static final TransportVersion V9_HANDSHAKE_VERSION = TransportVersion.fromId(8_800_00_0); + static final Set ALLOWED_HANDSHAKE_VERSIONS = Set.of( + EARLIEST_HANDSHAKE_VERSION, + REQUEST_HANDSHAKE_VERSION, + V9_HANDSHAKE_VERSION + ); static final String HANDSHAKE_ACTION_NAME = "internal:tcp/handshake"; private final ConcurrentMap pendingHandshakes = new ConcurrentHashMap<>(); diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index 8fa188efbd4a3..a4f606b54827b 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -7,21 +7,13 @@ # License v3.0 only", or the "Server Side Public License, v 1". # -org.elasticsearch.action.admin.indices.stats.IndicesStatsFeatures org.elasticsearch.action.bulk.BulkFeatures org.elasticsearch.features.FeatureInfrastructureFeatures -org.elasticsearch.health.HealthFeatures -org.elasticsearch.cluster.metadata.MetadataFeatures -org.elasticsearch.rest.RestFeatures -org.elasticsearch.repositories.RepositoriesFeatures -org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures org.elasticsearch.rest.action.admin.cluster.ClusterRerouteFeatures org.elasticsearch.index.IndexFeatures org.elasticsearch.index.mapper.MapperFeatures -org.elasticsearch.ingest.IngestGeoIpFeatures org.elasticsearch.search.SearchFeatures org.elasticsearch.search.retriever.RetrieversFeatures org.elasticsearch.script.ScriptFeatures -org.elasticsearch.reservedstate.service.FileSettingsFeatures org.elasticsearch.cluster.routing.RoutingFeatures org.elasticsearch.action.admin.cluster.stats.ClusterStatsFeatures diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java index f3d8f8860ba83..d9bf3e0e99c81 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/TransportGetAllocationStatsActionTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.routing.allocation.NodeAllocationStatsTests; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ClusterServiceUtils; @@ -35,8 +34,6 @@ import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.not; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -48,7 +45,6 @@ public class TransportGetAllocationStatsActionTests extends ESTestCase { private ClusterService clusterService; private TransportService transportService; private AllocationStatsService allocationStatsService; - private FeatureService featureService; private TransportGetAllocationStatsAction action; @@ -67,15 +63,13 @@ public void setUp() throws Exception { Set.of() ); allocationStatsService = mock(AllocationStatsService.class); - featureService = mock(FeatureService.class); action = new TransportGetAllocationStatsAction( transportService, clusterService, threadPool, new ActionFilters(Set.of()), null, - allocationStatsService, - featureService + allocationStatsService ); } @@ -99,8 +93,6 @@ public void testReturnsOnlyRequestedStats() throws Exception { ); when(allocationStatsService.stats()).thenReturn(Map.of(randomIdentifier(), NodeAllocationStatsTests.randomNodeAllocationStats())); - when(featureService.clusterHasFeature(any(ClusterState.class), eq(AllocationStatsFeatures.INCLUDE_DISK_THRESHOLD_SETTINGS))) - .thenReturn(true); var future = new PlainActionFuture(); action.masterOperation(mock(Task.class), request, ClusterState.EMPTY_STATE, future); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java index 8a51963097dae..9fef4c4ed328f 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusActionTests.java @@ -13,8 +13,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.indices.SystemIndexDescriptorUtils; import org.elasticsearch.indices.SystemIndices; @@ -31,7 +31,8 @@ public class TransportGetFeatureUpgradeStatusActionTests extends ESTestCase { public static String TEST_SYSTEM_INDEX_PATTERN = ".test*"; - private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(6000099); + // Version just before MINIMUM_COMPATIBLE in order to check that UpgradeStatus.MIGRATION_NEEDED is set correctly + private static final IndexVersion TEST_OLD_VERSION = IndexVersion.fromId(IndexVersions.MINIMUM_COMPATIBLE.id() - 1); private static final ClusterState CLUSTER_STATE = getClusterState(); private static final SystemIndices.Feature FEATURE = getFeature(); @@ -85,8 +86,6 @@ private static ClusterState getClusterState() { .numberOfReplicas(0) .build(); - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - // Once we start testing 9.x, we should update this test to use a 7.x "version created" IndexMetadata indexMetadata2 = IndexMetadata.builder(".test-index-2") .settings(Settings.builder().put("index.version.created", TEST_OLD_VERSION).build()) .numberOfShards(1) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java deleted file mode 100644 index 5954de0586985..0000000000000 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsSerializationTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.action.admin.cluster.settings; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -public class ClusterGetSettingsSerializationTests extends AbstractWireSerializingTestCase { - @Override - protected Writeable.Reader instanceReader() { - return ClusterGetSettingsAction.Response::new; - } - - @Override - protected ClusterGetSettingsAction.Response createTestInstance() { - final Settings persistentSettings = Settings.builder() - .put("persistent.foo.filtered", "bar") - .put("persistent.foo.non_filtered", "baz") - .build(); - - final Settings transientSettings = Settings.builder() - .put("transient.foo.filtered", "bar") - .put("transient.foo.non_filtered", "baz") - .build(); - - final Settings allSettings = Settings.builder().put(persistentSettings).put(transientSettings).build(); - - return new ClusterGetSettingsAction.Response(persistentSettings, transientSettings, allSettings); - } - - @Override - protected ClusterGetSettingsAction.Response mutateInstance(ClusterGetSettingsAction.Response instance) { - final Settings otherSettings = Settings.builder().put("random.setting", randomAlphaOfLength(randomIntBetween(1, 12))).build(); - return switch (between(0, 2)) { - case 0 -> new ClusterGetSettingsAction.Response(otherSettings, instance.transientSettings(), instance.settings()); - case 1 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), otherSettings, instance.settings()); - case 2 -> new ClusterGetSettingsAction.Response(instance.persistentSettings(), instance.transientSettings(), otherSettings); - default -> throw new IllegalStateException("Unexpected switch value"); - }; - } -} diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java index 47f3f71bcc11b..58341b7d7fb9a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/settings/ClusterGetSettingsTests.java @@ -12,17 +12,18 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.util.List; +import java.util.Map; import static org.mockito.Mockito.mock; @@ -54,10 +55,8 @@ public void testTransportFilters() throws Exception { TransportClusterGetSettingsAction action = new TransportClusterGetSettingsAction( transportService, mock(ClusterService.class), - threadPool, filter, - mock(ActionFilters.class), - mock(IndexNameExpressionResolver.class) + mock(ActionFilters.class) ); final Settings persistentSettings = Settings.builder() @@ -74,7 +73,8 @@ public void testTransportFilters() throws Exception { final ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE).metadata(metadata).build(); final PlainActionFuture future = new PlainActionFuture<>(); - action.masterOperation(null, null, clusterState, future); + final var task = new CancellableTask(1, "test", ClusterGetSettingsAction.NAME, "", null, Map.of()); + action.localClusterStateOperation(task, null, clusterState, future); assertTrue(future.isDone()); final ClusterGetSettingsAction.Response response = future.get(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index 96954458c18c4..0617053769da0 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -17,8 +17,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.script.Script; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; @@ -586,7 +586,7 @@ public void testSourceModes() { int numDisabledIndices = randomIntBetween(1, 5); for (int i = 0; i < numSyntheticIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-synthetic-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic") ); builder.put(indexMetadata); } @@ -594,7 +594,7 @@ public void testSourceModes() { IndexMetadata.Builder indexMetadata; if (randomBoolean()) { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "stored") ); } else { indexMetadata = new IndexMetadata.Builder("foo-stored-" + i).settings(indexSettings(IndexVersion.current(), 4, 1)); @@ -603,7 +603,7 @@ public void testSourceModes() { } for (int i = 0; i < numDisabledIndices; i++) { IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo-disabled-" + i).settings( - indexSettings(IndexVersion.current(), 4, 1).put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") + indexSettings(IndexVersion.current(), 4, 1).put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "disabled") ); builder.put(indexMetadata); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index f04180bde30f2..657a03066ada9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.query.QueryCacheStats; @@ -121,7 +120,6 @@ public class TransportRolloverActionTests extends ESTestCase { final DataStreamAutoShardingService dataStreamAutoShardingService = new DataStreamAutoShardingService( Settings.EMPTY, mockClusterService, - new FeatureService(List.of()), System::currentTimeMillis ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 02a4c134ccd15..1d861b340543d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -12,45 +12,21 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateIndexTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateIndexTemplateRequest::new; - } - - @Override - protected SimulateIndexTemplateRequest createTestInstance() { - SimulateIndexTemplateRequest req = new SimulateIndexTemplateRequest(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateIndexTemplateRequest mutateInstance(SimulateIndexTemplateRequest instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } +public class SimulateIndexTemplateRequestTests extends ESTestCase { public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest((String) null)); - expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest("")); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, null)); + expectThrows(IllegalArgumentException.class, () -> new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "")); } public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { @@ -60,7 +36,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest("testing"); + SimulateIndexTemplateRequest simulateRequest = new SimulateIndexTemplateRequest(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index 14ebf260d3bf9..6163566a5e593 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -12,49 +12,17 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; import java.util.List; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; -public class SimulateTemplateRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return SimulateTemplateAction.Request::new; - } - - @Override - protected SimulateTemplateAction.Request createTestInstance() { - SimulateTemplateAction.Request req = new SimulateTemplateAction.Request(randomAlphaOfLength(10)); - TransportPutComposableIndexTemplateAction.Request newTemplateRequest = new TransportPutComposableIndexTemplateAction.Request( - randomAlphaOfLength(4) - ); - newTemplateRequest.indexTemplate(ComposableIndexTemplateTests.randomInstance()); - req.indexTemplateRequest(newTemplateRequest); - req.includeDefaults(randomBoolean()); - return req; - } - - @Override - protected SimulateTemplateAction.Request mutateInstance(SimulateTemplateAction.Request instance) { - return randomValueOtherThan(instance, this::createTestInstance); - } - - public void testIndexNameCannotBeNullOrEmpty() { - expectThrows(IllegalArgumentException.class, () -> new SimulateTemplateAction.Request((String) null)); - expectThrows( - IllegalArgumentException.class, - () -> new SimulateTemplateAction.Request((TransportPutComposableIndexTemplateAction.Request) null) - ); - } +public class SimulateTemplateRequestTests extends ESTestCase { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); @@ -63,7 +31,7 @@ public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); - SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request("testing"); + SimulateTemplateAction.Request simulateRequest = new SimulateTemplateAction.Request(TEST_REQUEST_TIMEOUT, "testing"); simulateRequest.indexTemplateRequest(request); ActionRequestValidationException validationException = simulateRequest.validate(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 50885fc399c89..c6f923ce7cc03 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -153,7 +153,6 @@ class TestTransportBulkAction extends TransportBulkAction { transportService, TransportBulkActionIngestTests.this.clusterService, ingestService, - mockFeatureService, new NodeClient(Settings.EMPTY, TransportBulkActionIngestTests.this.threadPool), new ActionFilters(Collections.emptySet()), TestIndexNameExpressionResolver.newInstance(), diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index 587bc2e3ba333..616488d3472ff 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -113,7 +113,6 @@ class TestTransportBulkAction extends TransportBulkAction { transportService, TransportBulkActionTests.this.clusterService, null, - mockFeatureService, new NodeClient(Settings.EMPTY, TransportBulkActionTests.this.threadPool), new ActionFilters(Collections.emptySet()), new Resolver(), diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 2f033e4b5a383..544a373675ee5 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -247,7 +247,6 @@ static class TestTransportBulkAction extends TransportBulkAction { transportService, clusterService, null, - null, client, actionFilters, indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java b/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java index 3a43a1df9bf88..14ecf85b3aa7e 100644 --- a/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/datastreams/autosharding/DataStreamAutoShardingServiceTests.java @@ -27,9 +27,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.features.FeatureSpecification; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; @@ -81,12 +78,6 @@ public void setupService() { service = new DataStreamAutoShardingService( Settings.builder().put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true).build(), clusterService, - new FeatureService(List.of(new FeatureSpecification() { - @Override - public Set getFeatures() { - return Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE); - } - })), () -> now ); dataStreamName = randomAlphaOfLengthBetween(10, 100); @@ -113,14 +104,6 @@ public void testCalculateValidations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -129,12 +112,6 @@ public void testCalculateValidations() { DataStreamAutoShardingService disabledAutoshardingService = new DataStreamAutoShardingService( Settings.EMPTY, clusterService, - new FeatureService(List.of(new FeatureSpecification() { - @Override - public Set getFeatures() { - return Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE); - } - })), System::currentTimeMillis ); @@ -142,46 +119,6 @@ public Set getFeatures() { assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT)); } - { - // cluster doesn't have feature - ClusterState stateNoFeature = ClusterState.builder(ClusterName.DEFAULT) - .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures(Map.of("n1", Set.of(), "n2", Set.of())) - .metadata(Metadata.builder()) - .build(); - - Settings settings = Settings.builder().put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true).build(); - DataStreamAutoShardingService noFeatureService = new DataStreamAutoShardingService( - settings, - clusterService, - new FeatureService(List.of()), - () -> now - ); - - AutoShardingResult autoShardingResult = noFeatureService.calculate(stateNoFeature, dataStream, 2.0); - assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT)); - } - - { - Settings settings = Settings.builder() - .put(DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_ENABLED, true) - .putList( - DataStreamAutoShardingService.DATA_STREAMS_AUTO_SHARDING_EXCLUDES_SETTING.getKey(), - List.of("foo", dataStreamName + "*") - ) - .build(); - // patterns are configured to exclude the current data stream - DataStreamAutoShardingService noFeatureService = new DataStreamAutoShardingService( - settings, - clusterService, - new FeatureService(List.of()), - () -> now - ); - - AutoShardingResult autoShardingResult = noFeatureService.calculate(state, dataStream, 2.0); - assertThat(autoShardingResult, is(NOT_APPLICABLE_RESULT)); - } - { // null write load passed AutoShardingResult autoShardingResult = service.calculate(state, dataStream, null); @@ -209,14 +146,6 @@ public void testCalculateIncreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -248,14 +177,6 @@ public void testCalculateIncreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -287,14 +208,6 @@ public void testCalculateIncreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -326,14 +239,6 @@ public void testCalculateDecreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -367,14 +272,6 @@ public void testCalculateDecreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -416,14 +313,6 @@ public void testCalculateDecreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -463,14 +352,6 @@ public void testCalculateDecreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); @@ -504,14 +385,6 @@ public void testCalculateDecreaseShardingRecommendations() { builder.put(dataStream); ClusterState state = ClusterState.builder(ClusterName.DEFAULT) .nodes(DiscoveryNodes.builder().add(DiscoveryNodeUtils.create("n1")).add(DiscoveryNodeUtils.create("n2"))) - .nodeFeatures( - Map.of( - "n1", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()), - "n2", - Set.of(DataStreamAutoShardingService.DATA_STREAM_AUTO_SHARDING_FEATURE.id()) - ) - ) .metadata(builder) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 3623683532c59..3ed74392f746e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -561,6 +561,9 @@ public void testValidateIndexName() throws Exception { validateIndexName(checkerService, "..", "must not be '.' or '..'"); validateIndexName(checkerService, "foo:bar", "must not contain ':'"); + + validateIndexName(checkerService, "", "must not be empty"); + validateIndexName(checkerService, null, "must not be empty"); })); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java index 760900817780a..2c15addfe217b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/allocator/DesiredBalanceTests.java @@ -46,19 +46,19 @@ public void testShardMovements() { ); assertThat( - "1 shard movements when existing shard is moved and new shard copy is unassigned", + "1 shard movements when an existing shard copy is moved and new shard copy is unassigned", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 3, 1, 0)), equalTo(1) ); assertThat( - "1 shard movement", + "1 shard movement when an existing shard copy is moved", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("a", "c"), 2, 0, 0)), equalTo(1) ); assertThat( - "2 shard movement", + "2 shard movements when both shard copies are move to new nodes", shardMovements(new ShardAssignment(Set.of("a", "b"), 2, 0, 0), new ShardAssignment(Set.of("c", "d"), 2, 0, 0)), equalTo(2) ); @@ -77,10 +77,10 @@ public void testShardMovements() { } private static int shardMovements(ShardAssignment old, ShardAssignment updated) { - return DesiredBalance.shardMovements(of(old), of(updated)); + return DesiredBalance.shardMovements(createDesiredBalanceWith(old), createDesiredBalanceWith(updated)); } - private static DesiredBalance of(ShardAssignment assignment) { + private static DesiredBalance createDesiredBalanceWith(ShardAssignment assignment) { return new DesiredBalance(1, Map.of(new ShardId("index", "_na_", 0), assignment)); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java index 7c1c954e7b4e9..e6f50ef42365e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/ClusterApplierServiceTests.java @@ -66,6 +66,12 @@ public long relativeTimeInMillis() { assertThat(Thread.currentThread().getName(), containsString(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME)); return currentTimeMillis; } + + @Override + public long rawRelativeTimeInMillis() { + assertThat(Thread.currentThread().getName(), containsString(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME)); + return currentTimeMillis; + } }; clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); allowClusterStateApplicationFailure = false; @@ -207,15 +213,33 @@ public void testLongClusterStateUpdateLogging() throws Exception { ); mockLog.addExpectation( new MockLog.SeenEventExpectation( - "test4", + "test3", ClusterApplierService.class.getCanonicalName(), Level.WARN, "*cluster state applier task [test3] took [34s] which is above the warn threshold of [*]: " + "[running task [test3]] took [*" ) ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "test4", + ClusterApplierService.class.getCanonicalName(), + Level.WARN, + "*cluster state applier task [test4] took [36s] which is above the warn threshold of [*]: " + + "[running task [test4]] took [*" + ) + ); + mockLog.addExpectation( + new MockLog.SeenEventExpectation( + "test5", + ClusterApplierService.class.getCanonicalName(), + Level.WARN, + "*cluster state applier task [test5] took [38s] which is above the warn threshold of [*]: " + + "[running task [test5]] took [*" + ) + ); - final CountDownLatch latch = new CountDownLatch(4); + final CountDownLatch latch = new CountDownLatch(6); final CountDownLatch processedFirstTask = new CountDownLatch(1); currentTimeMillis = randomLongBetween(0L, Long.MAX_VALUE / 2); clusterApplierService.runOnApplierThread( @@ -266,9 +290,39 @@ public void onFailure(Exception e) { } } ); + clusterApplierService.runOnApplierThread("test4", Priority.HIGH, currentState -> { + // do nothing (testing that onResponse is included in timing) + }, new ActionListener<>() { + + @Override + public void onResponse(Void unused) { + advanceTime(TimeValue.timeValueSeconds(36).millis()); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + fail(); + } + }); + clusterApplierService.runOnApplierThread("test5", Priority.HIGH, currentState -> { + throw new IllegalArgumentException("Testing that onFailure is included in timing"); + }, new ActionListener<>() { + + @Override + public void onResponse(Void unused) { + fail(); + } + + @Override + public void onFailure(Exception e) { + advanceTime(TimeValue.timeValueSeconds(38).millis()); + latch.countDown(); + } + }); // Additional update task to make sure all previous logging made it to the loggerName // We don't check logging for this on since there is no guarantee that it will occur before our check - clusterApplierService.runOnApplierThread("test4", Priority.HIGH, currentState -> {}, new ActionListener<>() { + clusterApplierService.runOnApplierThread("test6", Priority.HIGH, currentState -> {}, new ActionListener<>() { @Override public void onResponse(Void ignored) { latch.countDown(); diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java index de1990361e766..8f62f8d33eacd 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.core.RestApiVersion; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -116,8 +115,6 @@ public static NewSubObject parse(XContentParser parser) { } } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) - @AwaitsFix(bugUrl = "this can be re-enabled once our rest api version is bumped to V_9") public void testNotCompatibleRequest() throws IOException { NamedXContentRegistry registry = new NamedXContentRegistry( List.of( diff --git a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java index 7a63934b4810b..eca30fdce2cf8 100644 --- a/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java +++ b/server/src/test/java/org/elasticsearch/health/node/LocalHealthMonitorTests.java @@ -24,8 +24,6 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.RelativeByteSizeValue; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.features.FeatureService; -import org.elasticsearch.health.HealthFeatures; import org.elasticsearch.health.HealthStatus; import org.elasticsearch.health.metadata.HealthMetadata; import org.elasticsearch.health.node.selection.HealthNode; @@ -119,18 +117,9 @@ public void setUp() throws Exception { client = mock(Client.class); - FeatureService featureService = new FeatureService(List.of(new HealthFeatures())); - mockHealthTracker = new MockHealthTracker(); - localHealthMonitor = LocalHealthMonitor.create( - Settings.EMPTY, - clusterService, - threadPool, - client, - featureService, - List.of(mockHealthTracker) - ); + localHealthMonitor = LocalHealthMonitor.create(Settings.EMPTY, clusterService, threadPool, client, List.of(mockHealthTracker)); } @After diff --git a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java index 51dadd8154549..3069589f9556c 100644 --- a/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutorTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.features.FeatureService; -import org.elasticsearch.health.HealthFeatures; import org.elasticsearch.persistent.PersistentTaskState; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; import org.elasticsearch.persistent.PersistentTasksService; @@ -78,7 +77,7 @@ public void setUp() throws Exception { localNodeId = clusterService.localNode().getId(); persistentTasksService = mock(PersistentTasksService.class); settings = Settings.builder().build(); - featureService = new FeatureService(List.of(new HealthFeatures())); + featureService = new FeatureService(List.of()); clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); } diff --git a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java index 4d97235c5fae5..61bd5323b5b43 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java +++ b/server/src/test/java/org/elasticsearch/index/codec/vectors/es816/ES816BinaryQuantizedVectorsWriter.java @@ -437,32 +437,35 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( float[] centroid, float cDotC ) throws IOException { - long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); - final IndexOutput tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "temp", - segmentWriteState.context - ); - final IndexOutput tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( - binarizedVectorData.getName(), - "score_temp", - segmentWriteState.context - ); - IndexInput binarizedDataInput = null; - IndexInput binarizedScoreDataInput = null; - boolean success = false; - int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); - BinaryQuantizer quantizer = new BinaryQuantizer( + final long vectorDataOffset = binarizedVectorData.alignFilePointer(Float.BYTES); + final int descritizedDimension = BQVectorUtils.discretize(fieldInfo.getVectorDimension(), 64); + final BinaryQuantizer quantizer = new BinaryQuantizer( fieldInfo.getVectorDimension(), descritizedDimension, fieldInfo.getVectorSimilarityFunction() ); + + IndexOutput tempQuantizedVectorData = null; + IndexOutput tempScoreQuantizedVectorData = null; + final DocsWithFieldSet docsWithField; + boolean success = false; + try { + tempQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "temp", + segmentWriteState.context + ); + tempScoreQuantizedVectorData = segmentWriteState.directory.createTempOutput( + binarizedVectorData.getName(), + "score_temp", + segmentWriteState.context + ); FloatVectorValues floatVectorValues = KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); if (fieldInfo.getVectorSimilarityFunction() == COSINE) { floatVectorValues = new NormalizedFloatVectorValues(floatVectorValues); } - DocsWithFieldSet docsWithField = writeBinarizedVectorAndQueryData( + docsWithField = writeBinarizedVectorAndQueryData( tempQuantizedVectorData, tempScoreQuantizedVectorData, floatVectorValues, @@ -470,13 +473,30 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( quantizer ); CodecUtil.writeFooter(tempQuantizedVectorData); - IOUtils.close(tempQuantizedVectorData); + CodecUtil.writeFooter(tempScoreQuantizedVectorData); + success = true; + } finally { + if (success) { + IOUtils.close(tempQuantizedVectorData, tempScoreQuantizedVectorData); + } else { + IOUtils.closeWhileHandlingException(tempQuantizedVectorData, tempScoreQuantizedVectorData); + if (tempQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempQuantizedVectorData.getName()); + } + if (tempScoreQuantizedVectorData != null) { + IOUtils.deleteFilesIgnoringExceptions(segmentWriteState.directory, tempScoreQuantizedVectorData.getName()); + } + } + } + + IndexInput binarizedDataInput = null; + IndexInput binarizedScoreDataInput = null; + success = false; + try { binarizedDataInput = segmentWriteState.directory.openInput(tempQuantizedVectorData.getName(), segmentWriteState.context); binarizedVectorData.copyBytes(binarizedDataInput, binarizedDataInput.length() - CodecUtil.footerLength()); - long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; + final long vectorDataLength = binarizedVectorData.getFilePointer() - vectorDataOffset; CodecUtil.retrieveChecksum(binarizedDataInput); - CodecUtil.writeFooter(tempScoreQuantizedVectorData); - IOUtils.close(tempScoreQuantizedVectorData); binarizedScoreDataInput = segmentWriteState.directory.openInput( tempScoreQuantizedVectorData.getName(), segmentWriteState.context @@ -490,10 +510,9 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( cDotC, docsWithField ); - success = true; final IndexInput finalBinarizedDataInput = binarizedDataInput; final IndexInput finalBinarizedScoreDataInput = binarizedScoreDataInput; - OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( + final OffHeapBinarizedVectorValues vectorValues = new OffHeapBinarizedVectorValues.DenseOffHeapVectorValues( fieldInfo.getVectorDimension(), docsWithField.cardinality(), centroid, @@ -503,7 +522,7 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( vectorsScorer, finalBinarizedDataInput ); - RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( + final RandomVectorScorerSupplier scorerSupplier = vectorsScorer.getRandomVectorScorerSupplier( fieldInfo.getVectorSimilarityFunction(), new OffHeapBinarizedQueryVectorValues( finalBinarizedScoreDataInput, @@ -513,22 +532,20 @@ private CloseableRandomVectorScorerSupplier mergeOneFieldToIndex( ), vectorValues ); + final String tempQuantizedVectorDataName = tempQuantizedVectorData.getName(); + final String tempScoreQuantizedVectorDataName = tempScoreQuantizedVectorData.getName(); + success = true; return new BinarizedCloseableRandomVectorScorerSupplier(scorerSupplier, vectorValues, () -> { IOUtils.close(finalBinarizedDataInput, finalBinarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, - tempQuantizedVectorData.getName(), - tempScoreQuantizedVectorData.getName() + tempQuantizedVectorDataName, + tempScoreQuantizedVectorDataName ); }); } finally { if (success == false) { - IOUtils.closeWhileHandlingException( - tempQuantizedVectorData, - tempScoreQuantizedVectorData, - binarizedDataInput, - binarizedScoreDataInput - ); + IOUtils.closeWhileHandlingException(binarizedDataInput, binarizedScoreDataInput); IOUtils.deleteFilesIgnoringExceptions( segmentWriteState.directory, tempQuantizedVectorData.getName(), diff --git a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java index 2a6c3428d6d45..a5d5d9b210e33 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/LuceneSyntheticSourceChangesSnapshotTests.java @@ -19,14 +19,12 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LuceneSyntheticSourceChangesSnapshotTests extends SearchBasedChangesSnapshotTests { @Override protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } diff --git a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java index b764bce464d15..d0455c14bd784 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/TranslogOperationAsserterTests.java @@ -24,8 +24,6 @@ import java.io.IOException; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class TranslogOperationAsserterTests extends EngineTestCase { @Override @@ -33,7 +31,7 @@ protected Settings indexSettings() { return Settings.builder() .put(super.indexSettings()) .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) - .put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); } @@ -57,10 +55,10 @@ EngineConfig engineConfig(boolean useSyntheticSource) { EngineConfig config = engine.config(); Settings.Builder settings = Settings.builder().put(config.getIndexSettings().getSettings()); if (useSyntheticSource) { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true); } else { - settings.put(INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.name()); settings.put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), false); } IndexMetadata imd = IndexMetadata.builder(config.getIndexSettings().getIndexMetadata()).settings(settings).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 907a1a15721dc..4b0fac2cf2e0f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -54,7 +54,7 @@ public void putMappings() { } public void testGetMappings() { - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings().get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT).get(); assertExpectedMappings(getMappingsResponse.mappings()); } @@ -71,7 +71,7 @@ public void testGetFieldMappings() { assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS); // double check that submitting the filtered mappings to an unfiltered index leads to the same get field mappings output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filtered = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filtered.getSourceAsMap())); GetFieldMappingsResponse response = indicesAdmin().prepareGetFieldMappings("test").setFields("*").get(); @@ -98,7 +98,7 @@ public void testFieldCapabilities() { assertFieldCaps(filtered, filteredFields); // double check that submitting the filtered mappings to an unfiltered index leads to the same field_caps output // as the one coming from a filtered index with same mappings - GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings("filtered").get(); + GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "filtered").get(); MappingMetadata filteredMapping = getMappingsResponse.getMappings().get("filtered"); assertAcked(indicesAdmin().prepareCreate("test").setMapping(filteredMapping.getSourceAsMap())); FieldCapabilitiesResponse test = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("test")).actionGet(); @@ -155,7 +155,7 @@ private void assertExpectedMappings(Map mappings) { private void assertMappingsAreValid(Map sourceAsMap) { // check that the returned filtered mappings are still valid mappings by submitting them and retrieving them back assertAcked(indicesAdmin().prepareCreate("test").setMapping(sourceAsMap)); - GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings("test").get(); + GetMappingsResponse testMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertEquals(1, testMappingsResponse.getMappings().size()); // the mappings are returned unfiltered for this index, yet they are the same as the previous ones that were returned filtered assertFiltered(testMappingsResponse.getMappings().get("test")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java index 8646e1b66dcb0..1ba5f423d4b03 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperConfigurationTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.index.DirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; @@ -130,7 +131,7 @@ private MapperService mapperServiceWithCustomSettings( for (var entry : customSettings.entrySet()) { settings.put(entry.getKey(), entry.getValue()); } - settings.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); + settings.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC); return createMapperService(settings.build(), mapping(mapping)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java index 6ea41763cbcc2..b669c717719b0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NonDynamicFieldMapperTestCase.java @@ -43,7 +43,7 @@ public void testCreateExplicitMappingSucceeds() throws Exception { """, getMapping()); var resp = client().admin().indices().prepareCreate("test").setMapping(mapping).get(); assertTrue(resp.isAcknowledged()); - var mappingsResp = client().admin().indices().prepareGetMappings("test").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); var mappingMetadata = mappingsResp.getMappings().get("test"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); @@ -149,7 +149,7 @@ public void testCreateExplicitMappingInIndexTemplateSucceeds() throws Exception var resp = client().prepareIndex("test1").setSource("field", "hello world").get(); assertThat(resp.status(), equalTo(RestStatus.CREATED)); - var mappingsResp = client().admin().indices().prepareGetMappings("test1").get(); + var mappingsResp = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test1").get(); var mappingMetadata = mappingsResp.getMappings().get("test1"); var fieldType = XContentMapValues.extractValue("properties.field.type", mappingMetadata.getSourceAsMap()); assertThat(fieldType, equalTo(getTypeName())); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java index b7693513a434d..bc560d94b8f52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java @@ -427,7 +427,7 @@ public void testRecoverySourceWitInvalidSettings() { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); IllegalArgumentException exc = expectThrows( @@ -470,7 +470,7 @@ public void testRecoverySourceWitInvalidSettings() { public void testRecoverySourceWithSyntheticSource() throws IOException { { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); DocumentMapper docMapper = mapperService.documentMapper(); @@ -480,7 +480,7 @@ public void testRecoverySourceWithSyntheticSource() throws IOException { } { Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC.toString()) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService(settings, topMapping(b -> {})); @@ -539,7 +539,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -549,7 +549,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -559,7 +559,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -571,7 +571,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -581,7 +581,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -591,7 +591,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage()); @@ -613,7 +613,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -635,7 +635,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); final MapperService mapperService = createMapperService(settings, mappings); @@ -657,7 +657,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { """; final Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field") .build(); var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings)); @@ -668,7 +668,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); final MapperService mapperService = createMapperService(settings, mappings); DocumentMapper docMapper = mapperService.documentMapper(); @@ -677,7 +677,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); @@ -686,7 +686,7 @@ public void testStandardIndexModeWithSourceModeSetting() throws IOException { { final XContentBuilder mappings = topMapping(b -> {}); final Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED) .build(); final MapperService mapperService = createMapperService(settings, mappings); final DocumentMapper docMapper = mapperService.documentMapper(); diff --git a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java index 15febaa8db2ab..8a84b0ec59b51 100644 --- a/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java +++ b/server/src/test/java/org/elasticsearch/readiness/ReadinessServiceTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.http.HttpInfo; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.HttpStats; -import org.elasticsearch.reservedstate.service.FileSettingsFeatures; import org.elasticsearch.reservedstate.service.FileSettingsService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLog; @@ -48,7 +47,6 @@ import java.nio.channels.ServerSocketChannel; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import static org.elasticsearch.cluster.metadata.ReservedStateErrorMetadata.ErrorKind.TRANSIENT; @@ -60,7 +58,6 @@ public class ReadinessServiceTests extends ESTestCase implements ReadinessClient private ThreadPool threadpool; private Environment env; private FakeHttpTransport httpTransport; - private static final Set nodeFeatures = Set.of(FileSettingsFeatures.FILE_SETTINGS_SUPPORTED.id()); private static Metadata emptyReservedStateMetadata; static { @@ -310,26 +307,6 @@ public void testFileSettingsUpdateError() throws Exception { readinessService.close(); } - public void testFileSettingsMixedCluster() throws Exception { - readinessService.start(); - - // initially the service isn't ready because initial cluster state has not been applied yet - assertFalse(readinessService.ready()); - - ClusterState noFileSettingsState = ClusterState.builder(noFileSettingsState()) - // the master node is upgraded to support file settings, but existing node2 is not - .nodeFeatures(Map.of(httpTransport.node.getId(), nodeFeatures)) - .build(); - ClusterChangedEvent event = new ClusterChangedEvent("test", noFileSettingsState, emptyState()); - readinessService.clusterChanged(event); - - // when upgrading from nodes before file settings exist, readiness should return true once a master is elected - assertTrue(readinessService.ready()); - - readinessService.stop(); - readinessService.close(); - } - private ClusterState emptyState() { return ClusterState.builder(new ClusterName("cluster")) .nodes( @@ -347,7 +324,6 @@ private ClusterState noFileSettingsState() { .masterNodeId(httpTransport.node.getId()) .localNodeId(httpTransport.node.getId()) ) - .nodeFeatures(Map.of(httpTransport.node.getId(), nodeFeatures, "node2", nodeFeatures)) .build(); } } diff --git a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java index 040ab9fd5c2e9..028438b5e9267 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java @@ -210,12 +210,15 @@ public void testObsoleteVersion() { assertThat( e.getMessage(), equalTo( - "A compatible version is required on both Content-Type and Accept headers if either one has requested a " - + "compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(PREVIOUS_VERSION) - + ", Content-Type=" - + contentTypeHeader(OBSOLETE_VERSION) + "Content-Type version must be either version " + + CURRENT_VERSION + + " or " + + PREVIOUS_VERSION + + ", but found " + + OBSOLETE_VERSION + + ". " + + "Content-Type=" + + acceptHeader(OBSOLETE_VERSION) ) ); } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 0f73c367ff2ef..1bcc89b68c141 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -574,11 +574,10 @@ public void testPreTagsWithoutPostTags() throws IOException { assertEquals("pre_tags are set but post_tags are not set", e.getCause().getCause().getMessage()); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/119723") public void testInvalidMaxAnalyzedOffset() throws IOException { XContentParseException e = expectParseThrows( XContentParseException.class, - "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -1) + "}" + "{ \"max_analyzed_offset\" : " + randomIntBetween(-100, -2) + "}" ); assertThat(e.getMessage(), containsString("[highlight] failed to parse field [" + MAX_ANALYZED_OFFSET_FIELD.toString() + "]")); assertThat(e.getCause().getMessage(), containsString("[max_analyzed_offset] must be a positive integer, or -1")); diff --git a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java index da28b0eff441f..2724b86f9acd4 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/KnnRetrieverBuilderParsingTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Predicates; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -88,10 +89,7 @@ protected KnnRetrieverBuilder createTestInstance() { protected KnnRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { return (KnnRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, - new RetrieverParserContext( - new SearchUsage(), - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED - ) + new RetrieverParserContext(new SearchUsage(), Predicates.never()) ); } diff --git a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java b/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java deleted file mode 100644 index 6448d11de2e47..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/retriever/RetrieverBuilderVersionTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.search.retriever; - -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.features.NodeFeature; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.json.JsonXContent; - -import java.io.IOException; -import java.util.List; - -/** Tests retrievers validate on their own {@link NodeFeature} */ -public class RetrieverBuilderVersionTests extends ESTestCase { - - public void testRetrieverVersions() throws IOException { - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ParsingException iae = expectThrows(ParsingException.class, () -> ssb.parseXContent(parser, true, nf -> false)); - assertEquals("Unknown key for a START_OBJECT in [retriever].", iae.getMessage()); - } - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ParsingException iae = expectThrows( - ParsingException.class, - () -> ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED) - ); - assertEquals("unknown retriever [standard]", iae.getMessage()); - } - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"standard\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ssb.parseXContent( - parser, - true, - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED - ); - } - - try (XContentParser parser = createParser(JsonXContent.jsonXContent, "{\"retriever\":{\"knn\":{}}}")) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ParsingException iae = expectThrows( - ParsingException.class, - () -> ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED) - ); - assertEquals("unknown retriever [knn]", iae.getMessage()); - } - - try ( - XContentParser parser = createParser( - JsonXContent.jsonXContent, - "{\"retriever\":{\"knn\":{\"field\": \"test\", \"k\": 2, \"num_candidates\": 5, \"query_vector\": [1, 2, 3]}}}" - ) - ) { - SearchSourceBuilder ssb = new SearchSourceBuilder(); - ssb.parseXContent( - parser, - true, - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == KnnRetrieverBuilder.KNN_RETRIEVER_SUPPORTED - ); - } - } - - @Override - protected NamedXContentRegistry xContentRegistry() { - return new NamedXContentRegistry(new SearchModule(Settings.EMPTY, List.of()).getNamedXContents()); - } -} diff --git a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java index eacd949077bc4..979c588089f5c 100644 --- a/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java +++ b/server/src/test/java/org/elasticsearch/search/retriever/StandardRetrieverBuilderParsingTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Predicates; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchNoneQueryBuilder; @@ -100,10 +101,7 @@ protected StandardRetrieverBuilder createTestInstance() { protected StandardRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { return (StandardRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, - new RetrieverParserContext( - new SearchUsage(), - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == StandardRetrieverBuilder.STANDARD_RETRIEVER_SUPPORTED - ) + new RetrieverParserContext(new SearchUsage(), Predicates.never()) ); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java index cc21ade314715..cadd9d5196f69 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RepositoryIntegrityHealthIndicatorServiceTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.features.FeatureService; import org.elasticsearch.health.Diagnosis; import org.elasticsearch.health.Diagnosis.Resource.Type; -import org.elasticsearch.health.HealthFeatures; import org.elasticsearch.health.HealthIndicatorDetails; import org.elasticsearch.health.HealthIndicatorResult; import org.elasticsearch.health.SimpleHealthIndicatorDetails; @@ -37,7 +36,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Stream; import static org.elasticsearch.cluster.node.DiscoveryNode.DISCOVERY_NODE_COMPARATOR; @@ -382,10 +380,7 @@ public void testMappedFieldsForTelemetry() { } private ClusterState createClusterStateWith(RepositoriesMetadata metadata) { - var features = Set.of(HealthFeatures.SUPPORTS_EXTENDED_REPOSITORY_INDICATOR.id()); - var builder = ClusterState.builder(new ClusterName("test-cluster")) - .nodes(DiscoveryNodes.builder().add(node1).add(node2).build()) - .nodeFeatures(Map.of(node1.getId(), features, node2.getId(), features)); + var builder = ClusterState.builder(new ClusterName("test-cluster")).nodes(DiscoveryNodes.builder().add(node1).add(node2).build()); if (metadata != null) { builder.metadata(Metadata.builder().putCustom(RepositoriesMetadata.TYPE, metadata)); } @@ -399,7 +394,7 @@ private static RepositoryMetadata createRepositoryMetadata(String name, boolean private RepositoryIntegrityHealthIndicatorService createRepositoryIntegrityHealthIndicatorService(ClusterState clusterState) { var clusterService = mock(ClusterService.class); when(clusterService.state()).thenReturn(clusterState); - return new RepositoryIntegrityHealthIndicatorService(clusterService, featureService); + return new RepositoryIntegrityHealthIndicatorService(clusterService); } private SimpleHealthIndicatorDetails createDetails(int total, int corruptedCount, List corrupted, int unknown, int invalid) { diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index 7a07e407024ce..0fe886f37aa47 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -241,9 +241,7 @@ import static org.hamcrest.Matchers.iterableWithSize; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class SnapshotResiliencyTests extends ESTestCase { @@ -2111,8 +2109,6 @@ protected void connectToNodesAndWait(ClusterState newClusterState) { } ); recoverySettings = new RecoverySettings(settings, clusterSettings); - FeatureService mockFeatureService = mock(FeatureService.class); - when(mockFeatureService.clusterHasFeature(any(), any())).thenReturn(true); mockTransport = new DisruptableMockTransport(node, deterministicTaskQueue) { @Override protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) { @@ -2403,7 +2399,6 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { null, FailureStoreMetrics.NOOP ), - mockFeatureService, client, actionFilters, indexNameExpressionResolver, diff --git a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java index 889afab00e830..c0185832d6122 100644 --- a/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java +++ b/server/src/test/java/org/elasticsearch/transport/InboundDecoderTests.java @@ -27,6 +27,7 @@ import static org.elasticsearch.common.bytes.ReleasableBytesReferenceStreamInputTests.wrapAsReleasable; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.instanceOf; @@ -182,7 +183,7 @@ public void testDecodePreHeaderSizeVariableInt() throws IOException { } } - public void testDecodeHandshakeCompatibility() throws IOException { + public void testDecodeHandshakeV7Compatibility() throws IOException { String action = "test-request"; long requestId = randomNonNegativeLong(); final String headerKey = randomAlphaOfLength(10); @@ -223,6 +224,55 @@ public void testDecodeHandshakeCompatibility() throws IOException { } + public void testDecodeHandshakeV8Compatibility() throws IOException { + doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, null); + doHandshakeCompatibilityTest(TransportHandshaker.REQUEST_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE); + } + + public void testDecodeHandshakeV9Compatibility() throws IOException { + doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, null); + doHandshakeCompatibilityTest(TransportHandshaker.V9_HANDSHAKE_VERSION, Compression.Scheme.DEFLATE); + } + + private void doHandshakeCompatibilityTest(TransportVersion transportVersion, Compression.Scheme compressionScheme) throws IOException { + String action = "test-request"; + long requestId = randomNonNegativeLong(); + final String headerKey = randomAlphaOfLength(10); + final String headerValue = randomAlphaOfLength(20); + threadContext.putHeader(headerKey, headerValue); + OutboundMessage message = new OutboundMessage.Request( + threadContext, + new TestRequest(randomAlphaOfLength(100)), + transportVersion, + action, + requestId, + true, + compressionScheme + ); + + try (RecyclerBytesStreamOutput os = new RecyclerBytesStreamOutput(recycler)) { + final BytesReference bytes = message.serialize(os); + int totalHeaderSize = TcpHeader.headerSize(transportVersion); + + InboundDecoder decoder = new InboundDecoder(recycler); + final ArrayList fragments = new ArrayList<>(); + final ReleasableBytesReference releasable1 = wrapAsReleasable(bytes); + int bytesConsumed = decoder.decode(releasable1, fragments::add); + assertThat(bytesConsumed, greaterThan(totalHeaderSize)); + assertTrue(releasable1.hasReferences()); + + final Header header = (Header) fragments.get(0); + assertEquals(requestId, header.getRequestId()); + assertEquals(transportVersion, header.getVersion()); + assertEquals(compressionScheme == Compression.Scheme.DEFLATE, header.isCompressed()); + assertTrue(header.isHandshake()); + assertTrue(header.isRequest()); + assertFalse(header.needsToReadVariableHeader()); + assertEquals(headerValue, header.getRequestHeaders().get(headerKey)); + fragments.clear(); + } + } + public void testClientChannelTypeFailsDecodingRequests() throws Exception { String action = "test-request"; long requestId = randomNonNegativeLong(); @@ -488,23 +538,16 @@ public void testCheckVersionCompatibility() { } public void testCheckHandshakeCompatibility() { - try { - InboundDecoder.checkHandshakeVersionCompatibility(randomFrom(TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS)); - } catch (IllegalStateException e) { - throw new AssertionError(e); - } + for (final var allowedHandshakeVersion : TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS) { + InboundDecoder.checkHandshakeVersionCompatibility(allowedHandshakeVersion); // should not throw - var invalid = TransportVersion.fromId(TransportHandshaker.EARLIEST_HANDSHAKE_VERSION.id() - 1); - try { - InboundDecoder.checkHandshakeVersionCompatibility(invalid); - fail(); - } catch (IllegalStateException expected) { + var invalid = TransportVersion.fromId(allowedHandshakeVersion.id() + randomFrom(-1, +1)); assertEquals( "Received message from unsupported version: [" + invalid + "] allowed versions are: " + TransportHandshaker.ALLOWED_HANDSHAKE_VERSIONS, - expected.getMessage() + expectThrows(IllegalStateException.class, () -> InboundDecoder.checkHandshakeVersionCompatibility(invalid)).getMessage() ); } } diff --git a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java index ace3db377664c..958132b3e4076 100644 --- a/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java +++ b/test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java @@ -194,6 +194,16 @@ private void assertCircuitBreaks(ThrowingRunnable r) throws IOException { ); } + private void assertFoldCircuitBreaks(ThrowingRunnable r) throws IOException { + ResponseException e = expectThrows(ResponseException.class, r); + Map map = responseAsMap(e.getResponse()); + logger.info("expected fold circuit breaking {}", map); + assertMap( + map, + matchesMap().entry("status", 400).entry("error", matchesMap().extraOk().entry("type", "fold_too_much_memory_exception")) + ); + } + private void assertParseFailure(ThrowingRunnable r) throws IOException { ResponseException e = expectThrows(ResponseException.class, r); Map map = responseAsMap(e.getResponse()); @@ -325,11 +335,23 @@ public void testManyConcatFromRow() throws IOException { assertManyStrings(resp, strings); } + /** + * Hits a circuit breaker by building many moderately long strings. + */ + public void testHugeManyConcatFromRow() throws IOException { + assertFoldCircuitBreaks( + () -> manyConcat( + "ROW a=9999999999999, b=99999999999999999, c=99999999999999999, d=99999999999999999, e=99999999999999999", + 5000 + ) + ); + } + /** * Fails to parse a huge huge query. */ public void testHugeHugeManyConcatFromRow() throws IOException { - assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 50000)); + assertParseFailure(() -> manyConcat("ROW a=9999, b=9999, c=9999, d=9999, e=9999", 6000)); } /** @@ -387,13 +409,20 @@ public void testHugeManyRepeat() throws IOException { * Returns many moderately long strings. */ public void testManyRepeatFromRow() throws IOException { - int strings = 10000; + int strings = 300; Response resp = manyRepeat("ROW a = 99", strings); assertManyStrings(resp, strings); } /** - * Fails to parse a huge huge query. + * Hits a circuit breaker by building many moderately long strings. + */ + public void testHugeManyRepeatFromRow() throws IOException { + assertFoldCircuitBreaks(() -> manyRepeat("ROW a = 99", 400)); + } + + /** + * Fails to parse a huge, huge query. */ public void testHugeHugeManyRepeatFromRow() throws IOException { assertParseFailure(() -> manyRepeat("ROW a = 99", 100000)); diff --git a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java index 3ee18d71a5a79..56e702d2f76a0 100644 --- a/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java +++ b/test/fixtures/minio-fixture/src/main/java/org/elasticsearch/test/fixtures/minio/MinioTestContainer.java @@ -15,7 +15,7 @@ public final class MinioTestContainer extends DockerEnvironmentAwareTestContainer { private static final int servicePort = 9000; - public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2021-03-01T04-20-55Z"; + public static final String DOCKER_BASE_IMAGE = "minio/minio:RELEASE.2024-12-18T13-15-44Z"; private final boolean enabled; public MinioTestContainer(boolean enabled, String accessKey, String secretKey, String bucketName) { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index bce88a2a8d2f7..b33abc2d781e8 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1180,7 +1180,7 @@ private void assertSyntheticSourceWithTranslogSnapshot(SyntheticSourceSupport su var firstExample = support.example(1); int maxDocs = randomIntBetween(20, 50); var settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .put(IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING.getKey(), true) .build(); var mapperService = createMapperService(getVersion(), settings, () -> true, mapping(b -> { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java index 9054dc6f94182..e2a24eac6f8ab 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestFeatureService.java @@ -91,6 +91,12 @@ public boolean clusterHasFeature(String featureId, boolean any) { } if (hasFeatureMetadata()) { + if (isRestApiCompatibilityTest()) { + // assume this is a feature that has been assumed, then removed in this version + // the feature is therefore logically present, but not specified by the cluster + return true; + } + throw new IllegalArgumentException( Strings.format( "Unknown feature %s: check the respective FeatureSpecification is provided both in module-info.java " @@ -102,6 +108,10 @@ public boolean clusterHasFeature(String featureId, boolean any) { return false; } + private static boolean isRestApiCompatibilityTest() { + return Boolean.parseBoolean(System.getProperty("tests.restCompat", "false")); + } + public static boolean hasFeatureMetadata() { return MetadataHolder.FEATURE_NAMES.isEmpty() == false; } diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 357c0fe74b5ce..031f8f908847c 100644 --- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.Maps; -import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction; import org.elasticsearch.test.rest.Stash; import org.elasticsearch.test.rest.TestFeatureService; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; @@ -300,41 +299,24 @@ public Optional clusterHasCapabilities( params.put("capabilities", capabilitiesString); } params.put("error_trace", "false"); // disable error trace - - if (clusterHasFeature(RestNodesCapabilitiesAction.LOCAL_ONLY_CAPABILITIES.id(), false) == false) { - // can only check the whole cluster - if (any) { - logger.warn( - "Cluster does not support checking individual nodes for capabilities," - + "check for [{} {}?{} {}] may be incorrect in mixed-version clusters", - method, - path, - parametersString, - capabilitiesString - ); - } - return checkCapability(NodeSelector.ANY, params); - } else { - // check each node individually - we can actually check any here - params.put("local_only", "true"); // we're calling each node individually - - // individually call each node, so we can control whether we do an 'any' or 'all' check - List nodes = clientYamlTestClient.getRestClient(NodeSelector.ANY).getNodes(); - - for (Node n : nodes) { - Optional nodeResult = checkCapability(new SpecificNodeSelector(n), params); - if (nodeResult.isEmpty()) { - return Optional.empty(); - } else if (any == nodeResult.get()) { - // either any == true and node has cap, - // or any == false (ie all) and this node does not have cap - return nodeResult; - } + params.put("local_only", "true"); // we're calling each node individually + + // individually call each node, so we can control whether we do an 'any' or 'all' check + List nodes = clientYamlTestClient.getRestClient(NodeSelector.ANY).getNodes(); + + for (Node n : nodes) { + Optional nodeResult = checkCapability(new SpecificNodeSelector(n), params); + if (nodeResult.isEmpty()) { + return Optional.empty(); + } else if (any == nodeResult.get()) { + // either any == true and node has cap, + // or any == false (ie all) and this node does not have cap + return nodeResult; } - - // if we got here, either any is true and no node has it, or any == false and all nodes have it - return Optional.of(any == false); } + + // if we got here, either any is true and no node has it, or any == false and all nodes have it + return Optional.of(any == false); } private Optional checkCapability(NodeSelector nodeSelector, Map params) { diff --git a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java index df1022e45c5a5..2357b80a402b3 100644 --- a/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java +++ b/x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/existence/FrozenExistenceDeciderIT.java @@ -123,7 +123,7 @@ public void testZeroToOne() throws Exception { assertBusy(() -> { ExplainLifecycleResponse response = client().execute( ExplainLifecycleAction.INSTANCE, - new ExplainLifecycleRequest().indices(INDEX_NAME) + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(INDEX_NAME) ).actionGet(); IndexLifecycleExplainResponse indexResponse = response.getIndexResponses().get(INDEX_NAME); assertNotNull(indexResponse); diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 0bb4afe51b85a..e8e19bad2a7ef 100644 --- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -368,7 +368,7 @@ public void testSyntheticSource() throws Exception { if ("leader".equals(targetCluster)) { logger.info("Running against leader cluster"); Settings settings = Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); createIndex(adminClient(), leaderIndexName, settings, """ "properties": {"kwd": {"type": "keyword"}}}""", null); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java index 7ce55313aa771..cca8228557aba 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/CcrRepositoryIT.java @@ -488,7 +488,7 @@ public void testFollowerMappingIsUpdated() throws IOException { clusterStateRequest.indices(followerIndex); MappingMetadata mappingMetadata = followerClient().admin() .indices() - .prepareGetMappings("index2") + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") .get() .getMappings() .get("index2"); diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java index 7556d02ddb1cf..e50627822e315 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/IndexFollowingIT.java @@ -371,7 +371,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs)); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); @@ -382,7 +387,12 @@ public void testSyncMappings() throws Exception { } assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), firstBatchNumDocs + secondBatchNumDocs)); - mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("integer")); assertThat(XContentMapValues.extractValue("properties.k.type", mappingMetadata.sourceAsMap()), equalTo("long")); pauseFollow("index2"); @@ -410,7 +420,12 @@ public void testNoMappingDefined() throws Exception { assertBusy(() -> assertHitCount(followerClient().prepareSearch("index2"), 1)); pauseFollow("index2"); - MappingMetadata mappingMetadata = followerClient().admin().indices().prepareGetMappings("index2").get().getMappings().get("index2"); + MappingMetadata mappingMetadata = followerClient().admin() + .indices() + .prepareGetMappings(TEST_REQUEST_TIMEOUT, "index2") + .get() + .getMappings() + .get("index2"); assertThat(XContentMapValues.extractValue("properties.f.type", mappingMetadata.sourceAsMap()), equalTo("long")); assertThat(XContentMapValues.extractValue("properties.k", mappingMetadata.sourceAsMap()), nullValue()); } @@ -1188,7 +1203,7 @@ public void testUpdateAnalysisLeaderIndexSettings() throws Exception { assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.type"), equalTo("custom")); assertThat(getSettingsResponse.getSetting("follower", "index.analysis.analyzer.my_analyzer.tokenizer"), equalTo("keyword")); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(TEST_REQUEST_TIMEOUT); getMappingsRequest.indices("follower"); GetMappingsResponse getMappingsResponse = followerClient().admin().indices().getMappings(getMappingsRequest).actionGet(); MappingMetadata mappingMetadata = getMappingsResponse.getMappings().get("follower"); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 357e1bca38e8f..6bc25215855ee 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.MapperTestUtils; import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ccr.Ccr; import org.elasticsearch.xpack.ccr.CcrSettings; @@ -335,7 +334,7 @@ public void testDynamicIndexSettingsAreClassified() { replicatedSettings.add(IndexSettings.PREFER_ILM_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING); replicatedSettings.add(IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_WRITE_SETTING); - replicatedSettings.add(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING); + replicatedSettings.add(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING); for (Setting setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) { // removed settings have no effect, they are only there for BWC diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java index 1bf52b663b30f..af7e223e1c5b9 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/snapshots/sourceonly/SourceOnlySnapshotIT.java @@ -197,7 +197,7 @@ public void testSnapshotWithDanglingLocalSegment() throws Exception { } private static void assertMappings(String sourceIdx, boolean requireRouting, boolean useNested) throws IOException { - GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(sourceIdx).get(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIdx).get(); MappingMetadata mapping = getMappingsResponse.getMappings().get(sourceIdx); String nested = useNested ? """ ,"incorrect":{"type":"object"},"nested":{"type":"nested","properties":{"value":{"type":"long"}}}""" : ""; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java index 5c607335bff39..beff1e6ab39de 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ExplainLifecycleRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.master.info.ClusterInfoRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.TimeValue; import java.io.IOException; import java.util.Arrays; @@ -27,8 +28,8 @@ public class ExplainLifecycleRequest extends ClusterInfoRequest onlyManaged = onlyManaged == false; default -> throw new AssertionError("Illegal randomisation branch"); } - ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest newRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); newRequest.indices(indices); newRequest.indicesOptions(indicesOptions); newRequest.onlyErrors(onlyErrors); diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java index 2e40481f7c561..48009f8105429 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; @@ -32,7 +31,6 @@ private NodesDeprecationCheckAction() { super(NAME); } - @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA) // this can be replaced with TransportRequest.Empty in v9 public static class NodeRequest extends TransportRequest { public NodeRequest() {} @@ -40,11 +38,6 @@ public NodeRequest() {} public NodeRequest(StreamInput in) throws IOException { super(in); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } } public static class NodeResponse extends BaseNodeResponse { diff --git a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java index 01a26a7a0b7bb..41d815c8206a7 100644 --- a/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java +++ b/x-pack/plugin/downsample/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java @@ -285,7 +285,7 @@ protected void masterOperation( // At any point if there is an issue, delete the downsample index // 1. Extract source index mappings - final GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(sourceIndexName); + final GetMappingsRequest getMappingsRequest = new GetMappingsRequest(request.masterNodeTimeout()).indices(sourceIndexName); getMappingsRequest.setParentTask(parentTask); client.admin().indices().getMappings(getMappingsRequest, listener.delegateFailureAndWrap((delegate, getMappingsResponse) -> { final Map sourceIndexMappings = getMappingsResponse.mappings() diff --git a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java index 33a436a1c7f1b..31b415270915b 100644 --- a/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java +++ b/x-pack/plugin/downsample/src/test/java/org/elasticsearch/xpack/downsample/DownsampleActionSingleNodeTests.java @@ -1179,7 +1179,7 @@ private InternalAggregations aggregate(final String index, AggregationBuilder ag @SuppressWarnings("unchecked") private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, DownsampleConfig config) throws Exception { // Retrieve field information for the metric fields - final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(sourceIndex).get(); + final GetMappingsResponse getMappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, sourceIndex).get(); final Map sourceIndexMappings = getMappingsResponse.mappings() .entrySet() .stream() @@ -1221,8 +1221,9 @@ private void assertDownsampleIndex(String sourceIndex, String downsampleIndex, D assertFieldMappings(config, metricFields, mappings); - GetMappingsResponse indexMappings = indicesAdmin().getMappings(new GetMappingsRequest().indices(downsampleIndex, sourceIndex)) - .actionGet(); + GetMappingsResponse indexMappings = indicesAdmin().getMappings( + new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(downsampleIndex, sourceIndex) + ).actionGet(); Map downsampleIndexProperties = (Map) indexMappings.mappings() .get(downsampleIndex) .sourceAsMap() diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java index 3081dcf11d95e..70293788c5a8f 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/rules/retriever/QueryRuleRetrieverBuilderTests.java @@ -53,10 +53,7 @@ protected QueryRuleRetrieverBuilder createTestInstance() { protected QueryRuleRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { return (QueryRuleRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, - new RetrieverParserContext( - new SearchUsage(), - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED - ) + new RetrieverParserContext(new SearchUsage(), nf -> nf == QueryRuleRetrieverBuilder.QUERY_RULE_RETRIEVERS_SUPPORTED) ); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java index 00765a8c0528c..b254612a700df 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expression.java @@ -78,12 +78,20 @@ public Expression(Source source, List children) { super(source, children); } - // whether the expression can be evaluated statically (folded) or not + /** + * Whether the expression can be evaluated statically, aka "folded", or not. + */ public boolean foldable() { return false; } - public Object fold() { + /** + * Evaluate this expression statically to a constant. It is an error to call + * this if {@link #foldable} returns false. + */ + public Object fold(FoldContext ctx) { + // TODO After removing FoldContext.unbounded from non-test code examine all calls + // for places we should use instanceof Literal instead throw new QlIllegalArgumentException("Should not fold expression"); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index 4e4338aad3704..739333ded0fde 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -107,10 +107,10 @@ public static boolean foldable(List exps) { return true; } - public static List fold(List exps) { + public static List fold(FoldContext ctx, List exps) { List folded = new ArrayList<>(exps.size()); for (Expression exp : exps) { - folded.add(exp.fold()); + folded.add(exp.fold(ctx)); } return folded; @@ -135,7 +135,7 @@ public static String name(Expression e) { /** * Is this {@linkplain Expression} guaranteed to have * only the {@code null} value. {@linkplain Expression}s that - * {@link Expression#fold()} to {@code null} may + * {@link Expression#fold} to {@code null} may * return {@code false} here, but should eventually be folded * into a {@link Literal} containing {@code null} which will return * {@code true} from here. diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java new file mode 100644 index 0000000000000..25da44c5fd226 --- /dev/null +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FoldContext.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.MemorySizeValue; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.xpack.esql.core.QlClientException; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Objects; + +/** + * Context passed to {@link Expression#fold}. This is not thread safe. + */ +public class FoldContext { + private static final long SMALL = MemorySizeValue.parseBytesSizeValueOrHeapRatio("5%", "small").getBytes(); + + /** + * {@link Expression#fold} using less than 5% of heap. Fine in tests but otherwise + * calling this is a signal that you either, shouldn't be calling {@link Expression#fold} + * at all, or should pass in a shared {@link FoldContext} made by {@code Configuration}. + */ + public static FoldContext small() { + return new FoldContext(SMALL); + } + + private final long initialAllowedBytes; + private long allowedBytes; + + public FoldContext(long allowedBytes) { + this.initialAllowedBytes = allowedBytes; + this.allowedBytes = allowedBytes; + } + + /** + * The maximum allowed bytes. {@link #allowedBytes()} will be the same as this + * for an unused context. + */ + public long initialAllowedBytes() { + return initialAllowedBytes; + } + + /** + * The remaining allowed bytes. + */ + long allowedBytes() { + return allowedBytes; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + FoldContext that = (FoldContext) o; + return initialAllowedBytes == that.initialAllowedBytes && allowedBytes == that.allowedBytes; + } + + @Override + public int hashCode() { + return Objects.hash(initialAllowedBytes, allowedBytes); + } + + @Override + public String toString() { + return "FoldContext[" + allowedBytes + '/' + initialAllowedBytes + ']'; + } + + /** + * Track an allocation. Best to call this before allocating + * if possible, but after is ok if the allocation is small. + *

+ * Note that, unlike {@link CircuitBreaker}, you don't have + * to free this allocation later. This is important because the query plan + * doesn't implement {@link Releasable} so it can't free + * consistently. But when you have to allocate big chunks of memory during + * folding and know that you are returning the memory it is kindest to + * call this with a negative number, effectively giving those bytes back. + *

+ */ + public void trackAllocation(Source source, long bytes) { + allowedBytes -= bytes; + assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used"; + if (allowedBytes < 0) { + throw new FoldTooMuchMemoryException(source, bytes, initialAllowedBytes); + } + } + + /** + * Adapt this into a {@link CircuitBreaker} suitable for building bounded local + * DriverContext. This is absolutely an abuse of the {@link CircuitBreaker} contract + * and only methods used by BlockFactory are implemented. And this'll throw a + * {@link FoldTooMuchMemoryException} instead of the standard {@link CircuitBreakingException}. + * This works for the common folding implementation though. + */ + public CircuitBreaker circuitBreakerView(Source source) { + return new CircuitBreaker() { + @Override + public void circuitBreak(String fieldName, long bytesNeeded) { + throw new UnsupportedOperationException(); + } + + @Override + public void addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException { + trackAllocation(source, bytes); + } + + @Override + public void addWithoutBreaking(long bytes) { + assert bytes <= 0 : "we only expect this to be used for deallocation"; + allowedBytes -= bytes; + assert allowedBytes <= initialAllowedBytes : "returned more bytes than it used"; + } + + @Override + public long getUsed() { + /* + * This isn't expected to be used by we can implement it so we may as + * well. Maybe it'll be useful for debugging one day. + */ + return initialAllowedBytes - allowedBytes; + } + + @Override + public long getLimit() { + /* + * This isn't expected to be used by we can implement it so we may as + * well. Maybe it'll be useful for debugging one day. + */ + return initialAllowedBytes; + } + + @Override + public double getOverhead() { + return 1.0; + } + + @Override + public long getTrippedCount() { + return 0; + } + + @Override + public String getName() { + return REQUEST; + } + + @Override + public Durability getDurability() { + throw new UnsupportedOperationException(); + } + + @Override + public void setLimitAndOverhead(long limit, double overhead) { + throw new UnsupportedOperationException(); + } + }; + } + + public static class FoldTooMuchMemoryException extends QlClientException { + protected FoldTooMuchMemoryException(Source source, long bytesForExpression, long initialAllowedBytes) { + super( + "line {}:{}: Folding query used more than {}. The expression that pushed past the limit is [{}] which needed {}.", + source.source().getLineNumber(), + source.source().getColumnNumber(), + ByteSizeValue.ofBytes(initialAllowedBytes), + source.text(), + ByteSizeValue.ofBytes(bytesForExpression) + ); + } + } +} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java index 601758bca5918..233113c3fe1b8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Foldables.java @@ -10,9 +10,9 @@ public abstract class Foldables { - public static Object valueOf(Expression e) { + public static Object valueOf(FoldContext ctx, Expression e) { if (e.foldable()) { - return e.fold(); + return e.fold(ctx); } throw new QlIllegalArgumentException("Cannot determine value for {}", e); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java index 53f559c5c82fe..afe616489d81d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java @@ -98,7 +98,7 @@ public boolean resolved() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return value; } @@ -138,7 +138,7 @@ public String nodeString() { * Utility method for creating a literal out of a foldable expression. * Throws an exception if the expression is not foldable. */ - public static Literal of(Expression foldable) { + public static Literal of(FoldContext ctx, Expression foldable) { if (foldable.foldable() == false) { throw new QlIllegalArgumentException("Foldable expression required for Literal creation; received unfoldable " + foldable); } @@ -147,7 +147,7 @@ public static Literal of(Expression foldable) { return (Literal) foldable; } - return new Literal(foldable.source(), foldable.fold(), foldable.dataType()); + return new Literal(foldable.source(), foldable.fold(ctx), foldable.dataType()); } public static Literal of(Expression source, Object value) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index b817ec17c7bda..842f3c0ddadd7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -133,6 +133,14 @@ public static TypeResolution isFoldable(Expression e, String operationName, Para return TypeResolution.TYPE_RESOLVED; } + /** + * Is this {@link Expression#foldable()} and not {@code null}. + * + * @deprecated instead of calling this, check for a {@link Literal} containing + * {@code null}. Foldable expressions will be folded by other rules, + * eventually, to a {@link Literal}. + */ + @Deprecated public static TypeResolution isNotNullAndFoldable(Expression e, String operationName, ParamOrdinal paramOrd) { TypeResolution resolution = isFoldable(e, operationName, paramOrd); @@ -140,7 +148,7 @@ public static TypeResolution isNotNullAndFoldable(Expression e, String operation return resolution; } - if (e.dataType() == DataType.NULL || e.fold() == null) { + if (e.dataType() == DataType.NULL || e.fold(FoldContext.small()) == null) { resolution = new TypeResolution( format( null, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java index 8704a42ed33e2..36517b1be9ce7 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; @@ -53,5 +54,5 @@ public boolean foldable() { } @Override - public abstract Object fold(); + public abstract Object fold(FoldContext ctx); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java index be5caedacd50a..bf5549b31e5fa 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/BinaryPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -29,8 +30,8 @@ protected BinaryPredicate(Source source, Expression left, Expression right, F fu @SuppressWarnings("unchecked") @Override - public R fold() { - return function().apply((T) left().fold(), (U) right().fold()); + public R fold(FoldContext ctx) { + return function().apply((T) left().fold(ctx), (U) right().fold(ctx)); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java index 5de09f40437c7..a4e4685f764e8 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/Range.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; @@ -99,23 +100,24 @@ public boolean foldable() { } // We cannot fold the bounds here; but if they're already literals, we can check if the range is always empty. - if (lower() instanceof Literal && upper() instanceof Literal) { - return areBoundariesInvalid(); + if (lower() instanceof Literal l && upper() instanceof Literal u) { + return areBoundariesInvalid(l.value(), u.value()); } } - return false; } @Override - public Object fold() { - if (areBoundariesInvalid()) { + public Object fold(FoldContext ctx) { + Object lowerValue = lower.fold(ctx); + Object upperValue = upper.fold(ctx); + if (areBoundariesInvalid(lowerValue, upperValue)) { return Boolean.FALSE; } - Object val = value.fold(); - Integer lowerCompare = BinaryComparison.compare(lower.fold(), val); - Integer upperCompare = BinaryComparison.compare(val, upper().fold()); + Object val = value.fold(ctx); + Integer lowerCompare = BinaryComparison.compare(lower.fold(ctx), val); + Integer upperCompare = BinaryComparison.compare(val, upper().fold(ctx)); boolean lowerComparsion = lowerCompare == null ? false : (includeLower ? lowerCompare <= 0 : lowerCompare < 0); boolean upperComparsion = upperCompare == null ? false : (includeUpper ? upperCompare <= 0 : upperCompare < 0); return lowerComparsion && upperComparsion; @@ -125,9 +127,7 @@ public Object fold() { * Check whether the boundaries are invalid ( upper < lower) or not. * If they are, the value does not have to be evaluated. */ - protected boolean areBoundariesInvalid() { - Object lowerValue = lower.fold(); - Object upperValue = upper.fold(); + protected boolean areBoundariesInvalid(Object lowerValue, Object upperValue) { if (DataType.isDateTime(value.dataType()) || DataType.isDateTime(lower.dataType()) || DataType.isDateTime(upper.dataType())) { try { if (upperValue instanceof String upperString) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java index c4983b49a6bc8..218f61856accc 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -56,8 +57,8 @@ protected TypeResolution resolveType() { } @Override - public Object fold() { - return apply(field().fold()); + public Object fold(FoldContext ctx) { + return apply(field().fold(ctx)); } private static Boolean apply(Object input) { diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java index 9879a1f5ffc29..f5542ff7c3de5 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -49,8 +50,8 @@ protected IsNotNull replaceChild(Expression newChild) { } @Override - public Object fold() { - return field().fold() != null && DataType.isNull(field().dataType()) == false; + public Object fold(FoldContext ctx) { + return DataType.isNull(field().dataType()) == false && field().fold(ctx) != null; } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java index d88945045b03e..bb85791a9f85e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -45,8 +46,8 @@ protected IsNull replaceChild(Expression newChild) { } @Override - public Object fold() { - return field().fold() == null || DataType.isNull(field().dataType()); + public Object fold(FoldContext ctx) { + return DataType.isNull(field().dataType()) || field().fold(ctx) == null; } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java index 9a8a14f320cd6..b0e79704f5fda 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/operator/arithmetic/Neg.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -53,8 +54,8 @@ protected TypeResolution resolveType() { } @Override - public Object fold() { - return Arithmetics.negate((Number) field().fold()); + public Object fold(FoldContext ctx) { + return Arithmetics.negate((Number) field().fold(ctx)); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java index 0f9116ade5a31..a4a0a6217161e 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/regex/RegexMatch.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate.regex; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -62,7 +63,7 @@ public boolean foldable() { } @Override - public Boolean fold() { + public Boolean fold(FoldContext ctx) { throw new UnsupportedOperationException(); } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java new file mode 100644 index 0000000000000..2080f4007777c --- /dev/null +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/FoldContextTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.expression; + +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import static org.hamcrest.Matchers.equalTo; + +public class FoldContextTests extends ESTestCase { + public void testEq() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomFoldContext(), this::copy, this::mutate); + } + + private FoldContext randomFoldContext() { + FoldContext ctx = new FoldContext(randomNonNegativeLong()); + if (randomBoolean()) { + ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes())); + } + return ctx; + } + + private FoldContext copy(FoldContext ctx) { + FoldContext copy = new FoldContext(ctx.initialAllowedBytes()); + copy.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - ctx.allowedBytes()); + return copy; + } + + private FoldContext mutate(FoldContext ctx) { + if (randomBoolean()) { + FoldContext differentInitial = new FoldContext(ctx.initialAllowedBytes() + 1); + differentInitial.trackAllocation(Source.EMPTY, differentInitial.initialAllowedBytes() - ctx.allowedBytes()); + assertThat(differentInitial.allowedBytes(), equalTo(ctx.allowedBytes())); + return differentInitial; + } else { + FoldContext differentAllowed = new FoldContext(ctx.initialAllowedBytes()); + long allowed = randomValueOtherThan(ctx.allowedBytes(), () -> randomLongBetween(0, ctx.initialAllowedBytes())); + differentAllowed.trackAllocation(Source.EMPTY, ctx.initialAllowedBytes() - allowed); + assertThat(differentAllowed.allowedBytes(), equalTo(allowed)); + return differentAllowed; + } + } + + public void testTrackAllocation() { + FoldContext ctx = new FoldContext(10); + ctx.trackAllocation(Source.synthetic("shouldn't break"), 10); + Exception e = expectThrows( + FoldContext.FoldTooMuchMemoryException.class, + () -> ctx.trackAllocation(Source.synthetic("should break"), 1) + ); + assertThat( + e.getMessage(), + equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 1b." + ) + ); + } + + public void testCircuitBreakerViewBreaking() { + FoldContext ctx = new FoldContext(10); + ctx.circuitBreakerView(Source.synthetic("shouldn't break")).addEstimateBytesAndMaybeBreak(10, "test"); + Exception e = expectThrows( + FoldContext.FoldTooMuchMemoryException.class, + () -> ctx.circuitBreakerView(Source.synthetic("should break")).addEstimateBytesAndMaybeBreak(1, "test") + ); + assertThat( + e.getMessage(), + equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 1b." + ) + ); + } + + public void testCircuitBreakerViewWithoutBreaking() { + FoldContext ctx = new FoldContext(10); + CircuitBreaker view = ctx.circuitBreakerView(Source.synthetic("shouldn't break")); + view.addEstimateBytesAndMaybeBreak(10, "test"); + view.addWithoutBreaking(-1); + assertThat(view.getUsed(), equalTo(9L)); + } + + public void testToString() { + // Random looking numbers are indeed random. Just so we have consistent numbers to assert on in toString. + FoldContext ctx = new FoldContext(123); + ctx.trackAllocation(Source.EMPTY, 22); + assertThat(ctx.toString(), equalTo("FoldContext[101/123]")); + } +} diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java index ed4c6282368ca..cd15ed5a94cfc 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/predicate/RangeTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.core.expression.predicate; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.DateUtils; @@ -211,7 +212,11 @@ public void testAreBoundariesInvalid() { (Boolean) test[7], ZoneId.systemDefault() ); - assertEquals("failed on test " + i + ": " + Arrays.toString(test), test[8], range.areBoundariesInvalid()); + assertEquals( + "failed on test " + i + ": " + Arrays.toString(test), + test[8], + range.areBoundariesInvalid(range.lower().fold(FoldContext.small()), range.upper().fold(FoldContext.small())) + ); } } @@ -226,5 +231,4 @@ private static DataType randomNumericType() { private static DataType randomTextType() { return randomFrom(KEYWORD, TEXT); } - } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java index d323174d2d3d9..91b0564a5b404 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/optimizer/OptimizerRulesTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -104,7 +105,7 @@ public void testFoldExcludingRangeToFalse() { Range r = rangeOf(fa, SIX, false, FIVE, true); assertTrue(r.foldable()); - assertEquals(Boolean.FALSE, r.fold()); + assertEquals(Boolean.FALSE, r.fold(FoldContext.small())); } // 6 < a <= 5.5 -> FALSE @@ -113,7 +114,7 @@ public void testFoldExcludingRangeWithDifferentTypesToFalse() { Range r = rangeOf(fa, SIX, false, L(5.5d), true); assertTrue(r.foldable()); - assertEquals(Boolean.FALSE, r.fold()); + assertEquals(Boolean.FALSE, r.fold(FoldContext.small())); } // Conjunction diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 9cbcd8d49b586..d7fa786a24e7c 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -283,8 +283,9 @@ tasks.named('checkstyleMain').configure { exclude { normalize(it.file.toString()).contains("src/main/generated") } } -def prop(Type, type, TYPE, BYTES, Array) { +def prop(Name, Type, type, TYPE, BYTES, Array) { return [ + "Name" : Name, "Type" : Type, "type" : type, "TYPE" : TYPE, @@ -296,15 +297,19 @@ def prop(Type, type, TYPE, BYTES, Array) { "double" : type == "double" ? "true" : "", "BytesRef" : type == "BytesRef" ? "true" : "", "boolean" : type == "boolean" ? "true" : "", + "nanosMillis" : Name == "NanosMillis" ? "true" : "", + "millisNanos" : Name == "MillisNanos" ? "true" : "", ] } tasks.named('stringTemplates').configure { - var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray") - var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray") - var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray") - var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "") - var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray") + var intProperties = prop("Int", "Int", "int", "INT", "Integer.BYTES", "IntArray") + var longProperties = prop("Long", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var nanosMillisProperties = prop("NanosMillis", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var millisNanosProperties = prop("MillisNanos", "Long", "long", "LONG", "Long.BYTES", "LongArray") + var doubleProperties = prop("Double", "Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray") + var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "") + var booleanProperties = prop("Boolean", "Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray") File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st") template { @@ -322,6 +327,16 @@ tasks.named('stringTemplates').configure { it.inputFile = inInputFile it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java" } + template { + it.properties = nanosMillisProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java" + } + template { + it.properties = millisNanosProperties + it.inputFile = inInputFile + it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java" + } template { it.properties = doubleProperties it.inputFile = inInputFile diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java index 66fd7d3ee5eb5..7e25fb29fdb78 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; @@ -61,6 +62,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -350,6 +352,10 @@ public String toString() { public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY)); + public static LogicalOptimizerContext unboundLogicalOptimizerContext() { + return new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small()); + } + public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)); public static final QueryBuilderResolver MOCK_QUERY_BUILDER_RESOLVER = new MockQueryBuilderResolver(); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec index 57c4fca6223e1..bb46c9d31f745 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec @@ -1450,3 +1450,47 @@ FROM employees cnt:long 19 ; + +implicit casting strings to dates for IN operator +FROM employees +| WHERE birth_date IN ("1953-04-20", "1958-10-31") +| KEEP emp_no, first_name; +ignoreOrder:true + +emp_no:integer | first_name:keyword +10006 | Anneke +10025 | Prasadram +; + +IN operator with null in list, finds match + +FROM employees +| EVAL x = NULL +| WHERE birth_date IN (TO_DATETIME("1958-02-19T00:00:00Z"), x) +| KEEP birth_date, first_name; + +birth_date:datetime | first_name:keyword +1958-02-19T00:00:00.000Z | Saniya +; + +IN operator with null in list, doesn't find match + +FROM employees +| EVAL x = NULL +| WHERE birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x) +| KEEP birth_date, first_name; + +birth_date:datetime | first_name:keyword +; + +IN operator with null in list, doesn't find match, EVAL to check value + +FROM employees +| EVAL x = NULL +| EVAL result = birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x) +| LIMIT 1 +| KEEP result; + +result:boolean +null +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index 4206d6b48699f..71a65f059528a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1161,3 +1161,68 @@ row dt = to_date_nanos(0::long) plus:date_nanos 1970-01-01T00:00:01.000Z ; + +Date Nanos IN constant date nanos +required_capability: date_nanos_in_operator +required_capability: to_date_nanos + +FROM date_nanos +| WHERE MV_FIRST(nanos) IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z")); +ignoreOrder:true + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Date Nanos IN constant date nanos, implicit casting +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +FROM date_nanos +| WHERE MV_FIRST(nanos) IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z"); +ignoreOrder:true + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Date Nanos IN date nanos field, implicit casting +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +FROM date_nanos +| WHERE "2023-10-23T13:55:01.543123456Z" IN (MV_FIRST(nanos)); + +millis:date | nanos:date_nanos | num:long +2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456 +; + +Date nanos IN millisecond date field +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an == +FROM date_nanos +| WHERE MV_FIRST(nanos) IN (TO_DATETIME("2023-10-23T12:27:28.948"), TO_DATETIME("2020-02-02")); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; + +Millisecond field IN date nanos constants +required_capability: date_nanos_in_operator +required_capability: to_date_nanos +required_capability: date_nanos_implicit_casting + +# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an == +FROM date_nanos +| WHERE MV_FIRST(millis) IN (TO_DATE_NANOS("2023-10-23T12:27:28.948"), TO_DATE_NANOS("2020-02-02")); + +millis:date | nanos:date_nanos | num:long +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000 +; diff --git a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 index efc2e36609902..4d4cb7d2caac4 100644 --- a/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 +++ b/x-pack/plugin/esql/src/main/antlr/EsqlBaseParser.g4 @@ -143,18 +143,9 @@ indexString ; metadata - : metadataOption - | deprecated_metadata - ; - -metadataOption : METADATA UNQUOTED_SOURCE (COMMA UNQUOTED_SOURCE)* ; -deprecated_metadata - : OPENING_BRACKET metadataOption CLOSING_BRACKET - ; - metricsCommand : DEV_METRICS indexPattern (COMMA indexPattern)* aggregates=aggFields? (BY grouping=fields)? ; diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java new file mode 100644 index 0000000000000..14ca1e9a768fb --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code X-InEvaluator.java.st} instead. + */ +public class InMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public InMillisNanosEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + LongBlock[] rhsBlocks = new LongBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (LongBlock) rhs[i].eval(page); + } + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + LongVector[] rhsVectors = new LongVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getLong(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getLong(p); + } + result.appendBoolean(In.processMillisNanos(null, null, lhsVector.getLong(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InMillisNanosEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InMillisNanosEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java new file mode 100644 index 0000000000000..278ecdba7d9e5 --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Warnings; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Arrays; +import java.util.BitSet; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. + * This class is generated. Edit {@code X-InEvaluator.java.st} instead. + */ +public class InNanosMillisEvaluator implements EvalOperator.ExpressionEvaluator { + private final Source source; + + private final EvalOperator.ExpressionEvaluator lhs; + + private final EvalOperator.ExpressionEvaluator[] rhs; + + private final DriverContext driverContext; + + private Warnings warnings; + + public InNanosMillisEvaluator( + Source source, + EvalOperator.ExpressionEvaluator lhs, + EvalOperator.ExpressionEvaluator[] rhs, + DriverContext driverContext + ) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + this.driverContext = driverContext; + } + + @Override + public Block eval(Page page) { + try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) { + LongBlock[] rhsBlocks = new LongBlock[rhs.length]; + try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) { + for (int i = 0; i < rhsBlocks.length; i++) { + rhsBlocks[i] = (LongBlock) rhs[i].eval(page); + } + LongVector lhsVector = lhsBlock.asVector(); + if (lhsVector == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + LongVector[] rhsVectors = new LongVector[rhs.length]; + for (int i = 0; i < rhsBlocks.length; i++) { + rhsVectors[i] = rhsBlocks[i].asVector(); + if (rhsVectors[i] == null) { + return eval(page.getPositionCount(), lhsBlock, rhsBlocks); + } + } + return eval(page.getPositionCount(), lhsVector, rhsVectors); + } + } + } + + private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + BitSet nulls = new BitSet(rhs.length); + BitSet mvs = new BitSet(rhs.length); + boolean foundMatch; + for (int p = 0; p < positionCount; p++) { + if (lhsBlock.isNull(p)) { + result.appendNull(); + continue; + } + if (lhsBlock.getValueCount(p) != 1) { + if (lhsBlock.getValueCount(p) > 1) { + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + } + result.appendNull(); + continue; + } + // unpack rhsBlocks into rhsValues + nulls.clear(); + mvs.clear(); + for (int i = 0; i < rhsBlocks.length; i++) { + if (rhsBlocks[i].isNull(p)) { + nulls.set(i); + continue; + } + if (rhsBlocks[i].getValueCount(p) > 1) { + mvs.set(i); + warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value")); + continue; + } + int o = rhsBlocks[i].getFirstValueIndex(p); + rhsValues[i] = rhsBlocks[i].getLong(o); + } + if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) { + result.appendNull(); + continue; + } + foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues); + if (foundMatch) { + result.appendBoolean(true); + } else { + if (nulls.cardinality() > 0) { + result.appendNull(); + } else { + result.appendBoolean(false); + } + } + } + return result.build(); + } + } + + private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) { + try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) { + long[] rhsValues = new long[rhs.length]; + for (int p = 0; p < positionCount; p++) { + // unpack rhsVectors into rhsValues + for (int i = 0; i < rhsVectors.length; i++) { + rhsValues[i] = rhsVectors[i].getLong(p); + } + result.appendBoolean(In.processNanosMillis(null, null, lhsVector.getLong(p), rhsValues)); + } + return result.build(); + } + } + + @Override + public String toString() { + return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs)); + } + + private Warnings warnings() { + if (warnings == null) { + this.warnings = Warnings.createWarnings( + driverContext.warningsMode(), + source.source().getLineNumber(), + source.source().getColumnNumber(), + source.text() + ); + } + return warnings; + } + + static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + private final EvalOperator.ExpressionEvaluator.Factory lhs; + private final EvalOperator.ExpressionEvaluator.Factory[] rhs; + + Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) { + this.source = source; + this.lhs = lhs; + this.rhs = rhs; + } + + @Override + public InNanosMillisEvaluator get(DriverContext context) { + EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) + .map(a -> a.get(context)) + .toArray(EvalOperator.ExpressionEvaluator[]::new); + return new InNanosMillisEvaluator(source, lhs.get(context), rhs, context); + } + + @Override + public String toString() { + return "InNanosMillisEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index a329e249ab556..1acc72d4ea33e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -487,6 +487,11 @@ public enum Cap { */ DATE_NANOS_AGGREGATIONS(), + /** + * Support the {@link org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In} operator for date nanos + */ + DATE_NANOS_IN_OPERATOR(), + /** * DATE_PARSE supports reading timezones */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 3d1bfdfd0ef42..a11b511cb83b7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -325,7 +326,7 @@ protected LogicalPlan rule(Enrich plan, AnalyzerContext context) { // the policy does not exist return plan; } - final String policyName = (String) plan.policyName().fold(); + final String policyName = (String) plan.policyName().fold(FoldContext.small() /* TODO remove me */); final var resolved = context.enrichResolution().getResolvedPolicy(policyName, plan.mode()); if (resolved != null) { var policy = new EnrichPolicy(resolved.matchType(), null, List.of(), resolved.matchField(), resolved.enrichFields()); @@ -1279,16 +1280,16 @@ private static boolean supportsStringImplicitCasting(DataType type) { private static UnresolvedAttribute unresolvedAttribute(Expression value, String type, Exception e) { String message = format( "Cannot convert string [{}] to [{}], error [{}]", - value.fold(), + value.fold(FoldContext.small() /* TODO remove me */), type, (e instanceof ParsingException pe) ? pe.getErrorMessage() : e.getMessage() ); - return new UnresolvedAttribute(value.source(), String.valueOf(value.fold()), message); + return new UnresolvedAttribute(value.source(), String.valueOf(value.fold(FoldContext.small() /* TODO remove me */)), message); } private static Expression castStringLiteralToTemporalAmount(Expression from) { try { - TemporalAmount result = maybeParseTemporalAmount(from.fold().toString().strip()); + TemporalAmount result = maybeParseTemporalAmount(from.fold(FoldContext.small() /* TODO remove me */).toString().strip()); if (result == null) { return from; } @@ -1304,7 +1305,11 @@ private static Expression castStringLiteral(Expression from, DataType target) { try { return isTemporalAmount(target) ? castStringLiteralToTemporalAmount(from) - : new Literal(from.source(), EsqlDataTypeConverter.convert(from.fold(), target), target); + : new Literal( + from.source(), + EsqlDataTypeConverter.convert(from.fold(FoldContext.small() /* TODO remove me */), target), + target + ); } catch (Exception e) { return unresolvedAttribute(from, target.toString(), e); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java new file mode 100644 index 0000000000000..e3628d2554a78 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/PostOptimizationVerificationAware.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.capabilities; + +import org.elasticsearch.xpack.esql.common.Failures; +import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; + +/** + * Interface implemented by expressions that require validation post logical optimization, + * when the plan and references have been not just resolved but also replaced. + */ +public interface PostOptimizationVerificationAware { + + /** + * Validates the implementing expression - discovered failures are reported to the given + * {@link Failures} class. + * + *

+ * Example: the {@link Bucket} function, which produces buckets over a numerical or date field, based on a number of literal + * arguments needs to check if its arguments are all indeed literals. This is how this verification is performed: + * 

+     *     {@code
+     *
+     *      @Override
+     *      public void postLogicalOptimizationVerification(Failures failures) {
+     *          String operation = sourceText();
+     *
+     *          failures.add(isFoldable(buckets, operation, SECOND))
+     *              .add(from != null ? isFoldable(from, operation, THIRD) : null)
+     *              .add(to != null ? isFoldable(to, operation, FOURTH) : null);
+     *      }
+     *     }
+     *
+ * + */ + void postLogicalOptimizationVerification(Failures failures); +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java deleted file mode 100644 index f6733fa3f175c..0000000000000 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/capabilities/Validatable.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.capabilities; - -import org.elasticsearch.xpack.esql.common.Failures; - -/** - * Interface implemented by expressions that require validation post logical optimization, - * when the plan and references have been not just resolved but also replaced. - */ -public interface Validatable { - - /** - * Validates the implementing expression - discovered failures are reported to the given - * {@link Failures} class. - */ - default void validate(Failures failures) {} -} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java index 9a2e9398f52fd..b9c2b92ea72dd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/EvalMapper.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -50,13 +51,23 @@ public final class EvalMapper { private EvalMapper() {} @SuppressWarnings({ "rawtypes", "unchecked" }) - public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) { + public static ExpressionEvaluator.Factory toEvaluator(FoldContext foldCtx, Expression exp, Layout layout) { if (exp instanceof EvaluatorMapper m) { - return m.toEvaluator(e -> toEvaluator(e, layout)); + return m.toEvaluator(new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return toEvaluator(foldCtx, expression, layout); + } + + @Override + public FoldContext foldCtx() { + return foldCtx; + } + }); } for (ExpressionMapper em : MAPPERS) { if (em.typeToken.isInstance(exp)) { - return em.map(exp, layout); + return em.map(foldCtx, exp, layout); } } throw new QlIllegalArgumentException("Unsupported expression [{}]", exp); @@ -64,9 +75,9 @@ public static ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout lay static class BooleanLogic extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(BinaryLogic bc, Layout layout) { - var leftEval = toEvaluator(bc.left(), layout); - var rightEval = toEvaluator(bc.right(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, BinaryLogic bc, Layout layout) { + var leftEval = toEvaluator(foldCtx, bc.left(), layout); + var rightEval = toEvaluator(foldCtx, bc.right(), layout); /** * Evaluator for the three-valued boolean expressions. * We can't generate these with the {@link Evaluator} annotation because that @@ -142,8 +153,8 @@ public void close() { static class Nots extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Not not, Layout layout) { - var expEval = toEvaluator(not.field(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Not not, Layout layout) { + var expEval = toEvaluator(foldCtx, not.field(), layout); return dvrCtx -> new org.elasticsearch.xpack.esql.evaluator.predicate.operator.logical.NotEvaluator( not.source(), expEval.get(dvrCtx), @@ -154,7 +165,7 @@ public ExpressionEvaluator.Factory map(Not not, Layout layout) { static class Attributes extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Attribute attr, Layout layout) { + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Attribute attr, Layout layout) { record Attribute(int channel) implements ExpressionEvaluator { @Override public Block eval(Page page) { @@ -189,7 +200,7 @@ public boolean eagerEvalSafeInLazy() { static class Literals extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(Literal lit, Layout layout) { + public ExpressionEvaluator.Factory map(FoldContext foldCtx, Literal lit, Layout layout) { record LiteralsEvaluator(DriverContext context, Literal lit) implements ExpressionEvaluator { @Override public Block eval(Page page) { @@ -246,8 +257,8 @@ private static Block block(Literal lit, BlockFactory blockFactory, int positions static class IsNulls extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(IsNull isNull, Layout layout) { - var field = toEvaluator(isNull.field(), layout); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNull isNull, Layout layout) { + var field = toEvaluator(foldCtx, isNull.field(), layout); return new IsNullEvaluatorFactory(field); } @@ -294,8 +305,8 @@ public String toString() { static class IsNotNulls extends ExpressionMapper { @Override - public ExpressionEvaluator.Factory map(IsNotNull isNotNull, Layout layout) { - return new IsNotNullEvaluatorFactory(toEvaluator(isNotNull.field(), layout)); + public ExpressionEvaluator.Factory map(FoldContext foldCtx, IsNotNull isNotNull, Layout layout) { + return new IsNotNullEvaluatorFactory(toEvaluator(foldCtx, isNotNull.field(), layout)); } record IsNotNullEvaluatorFactory(EvalOperator.ExpressionEvaluator.Factory field) implements ExpressionEvaluator.Factory { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java index d8692faef5290..5a8b3d32e7db0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapper.java @@ -7,11 +7,20 @@ package org.elasticsearch.xpack.esql.evaluator.mapper; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.indices.breaker.AllCircuitBreakerStats; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.indices.breaker.CircuitBreakerStats; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.evaluator.EvalMapper; import org.elasticsearch.xpack.esql.planner.Layout; import static org.elasticsearch.compute.data.BlockUtils.fromArrayRow; @@ -23,9 +32,12 @@ public interface EvaluatorMapper { interface ToEvaluator { ExpressionEvaluator.Factory apply(Expression expression); + + FoldContext foldCtx(); } /** + * Convert this into an {@link ExpressionEvaluator}. *

* Note for implementors: * If you are implementing this function, you should call the passed-in @@ -35,8 +47,8 @@ interface ToEvaluator { *

* Note for Callers: * If you are attempting to call this method, and you have an - * {@link Expression} and a {@link org.elasticsearch.xpack.esql.planner.Layout}, - * you likely want to call {@link org.elasticsearch.xpack.esql.evaluator.EvalMapper#toEvaluator(Expression, Layout)} + * {@link Expression} and a {@link Layout}, + * you likely want to call {@link EvalMapper#toEvaluator} * instead. On the other hand, if you already have something that * looks like the parameter for this method, you should call this method * with that function. @@ -56,19 +68,89 @@ interface ToEvaluator { /** * Fold using {@link #toEvaluator} so you don't need a "by hand" - * implementation of fold. The evaluator that it makes is "funny" - * in that it'll always call {@link Expression#fold}, but that's - * good enough. + * implementation of {@link Expression#fold}. */ - default Object fold() { - return toJavaObject(toEvaluator(e -> driverContext -> new ExpressionEvaluator() { + default Object fold(Source source, FoldContext ctx) { + /* + * OK! So! We're going to build a bunch of *stuff* that so that we can + * call toEvaluator and use it without standing up an entire compute + * engine. + * + * Step 1 is creation of a `toEvaluator` which we'll soon use to turn + * the *children* of this Expression into ExpressionEvaluators. They + * have to be foldable or else we wouldn't have ended up here. So! + * We just call `fold` on them and turn the result of that into a + * Block. + * + * If the tree of expressions is pretty deep that `fold` call will + * likely end up being implemented by calling this method for the + * child. That's fine. Recursion is how you process trees. + */ + ToEvaluator foldChildren = new ToEvaluator() { @Override - public Block eval(Page page) { - return fromArrayRow(driverContext.blockFactory(), e.fold())[0]; + public ExpressionEvaluator.Factory apply(Expression expression) { + return driverContext -> new ExpressionEvaluator() { + @Override + public Block eval(Page page) { + return fromArrayRow(driverContext.blockFactory(), expression.fold(ctx))[0]; + } + + @Override + public void close() {} + }; } @Override - public void close() {} - }).get(DriverContext.getLocalDriver()).eval(new Page(1)), 0); + public FoldContext foldCtx() { + return ctx; + } + }; + + /* + * Step 2 is to create a DriverContext that we can pass to the above. + * This DriverContext is mostly about delegating to the FoldContext. + * That'll cause us to break if we attempt to allocate a huge amount + * of memory. Neat. + * + * Specifically, we make a CircuitBreaker view of the FoldContext, then + * we wrap it in a CircuitBreakerService so we can feed it to a BigArray + * so we can feed *that* into a DriverContext. It's a bit hacky, but + * that's what's going on here. + */ + CircuitBreaker breaker = ctx.circuitBreakerView(source); + BigArrays bigArrays = new BigArrays(null, new CircuitBreakerService() { + @Override + public CircuitBreaker getBreaker(String name) { + if (name.equals(CircuitBreaker.REQUEST) == false) { + throw new UnsupportedOperationException(); + } + return breaker; + } + + @Override + public AllCircuitBreakerStats stats() { + throw new UnsupportedOperationException(); + } + + @Override + public CircuitBreakerStats stats(String name) { + throw new UnsupportedOperationException(); + } + }, CircuitBreaker.REQUEST).withCircuitBreaking(); + DriverContext driverCtx = new DriverContext(bigArrays, new BlockFactory(breaker, bigArrays)); + + /* + * Finally we can call toEvaluator on ourselves! It'll fold our children, + * convert the result into Blocks, and then we'll run that with the memory + * breaking DriverContext. + * + * Then, finally finally, we turn the result into a java object to be compatible + * with the signature of `fold`. + */ + Block block = toEvaluator(foldChildren).get(driverCtx).eval(new Page(1)); + if (block.getPositionCount() != 1) { + throw new IllegalStateException("generated odd block from fold [" + block + "]"); + } + return toJavaObject(block, 0); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java index 5cd830058573f..5a76080e7995c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/evaluator/mapper/ExpressionMapper.java @@ -9,6 +9,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; import org.elasticsearch.xpack.esql.planner.Layout; @@ -19,5 +20,5 @@ public ExpressionMapper() { typeToken = ReflectionUtils.detectSuperTypeForRuleLike(getClass()); } - public abstract ExpressionEvaluator.Factory map(E expression, Layout layout); + public abstract ExpressionEvaluator.Factory map(FoldContext foldCtx, E expression, Layout layout); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java index 974f029eab2ef..94913581f696d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; import org.elasticsearch.xpack.esql.analysis.Verifier; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; @@ -56,6 +57,7 @@ public void esql( EsqlQueryRequest request, String sessionId, Configuration cfg, + FoldContext foldContext, EnrichPolicyResolver enrichPolicyResolver, EsqlExecutionInfo executionInfo, IndicesExpressionGrouper indicesExpressionGrouper, @@ -71,7 +73,7 @@ public void esql( enrichPolicyResolver, preAnalyzer, functionRegistry, - new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg)), + new LogicalPlanOptimizer(new LogicalOptimizerContext(cfg, foldContext)), mapper, verifier, planningMetrics, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java index 011fcaccf7fe4..8aa7f697489c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateFunction.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; @@ -92,7 +93,8 @@ public List parameters() { } public boolean hasFilter() { - return filter != null && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold()) == false); + return filter != null + && (filter.foldable() == false || Boolean.TRUE.equals(filter.fold(FoldContext.small() /* TODO remove me */)) == false); } public Expression filter() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java index 7436db9e00dd2..3170ae8f132c2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinct.java @@ -19,6 +19,7 @@ import org.elasticsearch.compute.aggregation.CountDistinctLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -210,7 +211,9 @@ protected TypeResolution resolveType() { @Override public AggregatorFunctionSupplier supplier(List inputChannels) { DataType type = field().dataType(); - int precision = this.precision == null ? DEFAULT_PRECISION : ((Number) this.precision.fold()).intValue(); + int precision = this.precision == null + ? DEFAULT_PRECISION + : ((Number) this.precision.fold(FoldContext.small() /* TODO remove me */)).intValue(); if (SUPPLIERS.containsKey(type) == false) { // If the type checking did its job, this should never happen throw EsqlIllegalArgumentException.illegalDataType(type); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java index 0d57267da1e29..8c943c991d501 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Percentile.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.aggregation.PercentileIntAggregatorFunctionSupplier; import org.elasticsearch.compute.aggregation.PercentileLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -170,7 +171,7 @@ protected AggregatorFunctionSupplier doubleSupplier(List inputChannels) } private int percentileValue() { - return ((Number) percentile.fold()).intValue(); + return ((Number) percentile.fold(FoldContext.small() /* TODO remove me */)).intValue(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java index 87ac9b77a6826..85ae65b6c5dc3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Rate.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -156,7 +157,7 @@ long unitInMillis() { } final Object foldValue; try { - foldValue = unit.fold(); + foldValue = unit.fold(FoldContext.small() /* TODO remove me */); } catch (Exception e) { throw new IllegalArgumentException("function [" + sourceText() + "] has invalid unit [" + unit.sourceText() + "]"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java index 40777b4d78dc2..9be8c94266ee8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Top.java @@ -21,6 +21,7 @@ import org.elasticsearch.compute.aggregation.TopLongAggregatorFunctionSupplier; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -115,11 +116,11 @@ Expression orderField() { } private int limitValue() { - return (int) limitField().fold(); + return (int) limitField().fold(FoldContext.small() /* TODO remove me */); } private String orderRawValue() { - return BytesRefs.toString(orderField().fold()); + return BytesRefs.toString(orderField().fold(FoldContext.small() /* TODO remove me */)); } private boolean orderValue() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java index 49c68d002440f..bab65653ba576 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/WeightedAvg.java @@ -11,8 +11,8 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -34,7 +34,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; -public class WeightedAvg extends AggregateFunction implements SurrogateExpression, Validatable { +public class WeightedAvg extends AggregateFunction implements SurrogateExpression { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "WeightedAvg", @@ -115,9 +115,15 @@ protected Expression.TypeResolution resolveType() { return resolution; } - if (weight.dataType() == DataType.NULL - || (weight.foldable() && (weight.fold() == null || weight.fold().equals(0) || weight.fold().equals(0.0)))) { - return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weight.foldable() ? weight.fold() : null)); + if (weight.dataType() == DataType.NULL) { + return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), null)); + } + if (weight.foldable() == false) { + return TypeResolution.TYPE_RESOLVED; + } + Object weightVal = weight.fold(FoldContext.small()/* TODO remove me*/); + if (weightVal == null || weightVal.equals(0) || weightVal.equals(0.0)) { + return new TypeResolution(format(null, invalidWeightError, SECOND, sourceText(), weightVal)); } return TypeResolution.TYPE_RESOLVED; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 07c4bb282ba71..4da7c01139c24 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -111,7 +112,7 @@ public Expression query() { * @return query expression as an object */ public Object queryAsObject() { - Object queryAsObject = query().fold(); + Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */); if (queryAsObject instanceof BytesRef bytesRef) { return bytesRef.utf8ToString(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java index fb9f8cff1b8b0..93f23d2f7ad08 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Match.java @@ -13,11 +13,12 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.planner.ExpressionTranslator; import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery; @@ -60,7 +61,7 @@ /** * Full text function that performs a {@link QueryStringQuery} . */ -public class Match extends FullTextFunction implements Validatable { +public class Match extends FullTextFunction implements PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Match", Match::readFrom); @@ -102,7 +103,8 @@ public class Match extends FullTextFunction implements Validatable { Use `MATCH` to perform a <> on the specified field. Using `MATCH` is equivalent to using the `match` query in the Elasticsearch Query DSL. - Match can be used on text fields, as well as other field types like boolean, dates, and numeric types. + Match can be used on fields from the text family like <> and <>, + as well as other field types like keyword, boolean, dates, and numeric types. For a simplified syntax, you can use the <> `:` operator instead of `MATCH`. @@ -201,7 +203,7 @@ protected TypeResolution checkParamCompatibility() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { Expression fieldExpression = field(); // Field may be converted to other data type (field_name :: data_type), so we need to check the original field if (fieldExpression instanceof AbstractConvertFunction convertFunction) { @@ -222,7 +224,7 @@ public void validate(Failures failures) { @Override public Object queryAsObject() { - Object queryAsObject = query().fold(); + Object queryAsObject = query().fold(FoldContext.small() /* TODO remove me */); // Convert BytesRef to string for string-based values if (queryAsObject instanceof BytesRef bytesRef) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java index ff8085cd1b44b..e77f95073050f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/Term.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -39,7 +39,7 @@ /** * Full text function that performs a {@link TermQuery} . */ -public class Term extends FullTextFunction implements Validatable { +public class Term extends FullTextFunction implements PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Term", Term::readFrom); @@ -100,7 +100,7 @@ protected TypeResolution resolveNonQueryParamTypes() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { if (field instanceof FieldAttribute == false) { failures.add( Failure.fail( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java index 12932ba8d6e11..7a3e080f5c830 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Bucket.java @@ -15,9 +15,10 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Foldables; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -56,7 +57,7 @@ * from a number of desired buckets (as a hint) and a range (auto mode). * In the former case, two parameters will be provided, in the latter four. */ -public class Bucket extends GroupingFunction implements Validatable, TwoOptionalArguments { +public class Bucket extends GroupingFunction implements PostOptimizationVerificationAware, TwoOptionalArguments { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Bucket", Bucket::new); // TODO maybe we should just cover the whole of representable dates here - like ten years, 100 years, 1000 years, all the way up. @@ -255,25 +256,25 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.dataType() == DataType.DATETIME || field.dataType() == DataType.DATE_NANOS) { Rounding.Prepared preparedRounding; if (buckets.dataType().isWholeNumber()) { - int b = ((Number) buckets.fold()).intValue(); - long f = foldToLong(from); - long t = foldToLong(to); + int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue(); + long f = foldToLong(toEvaluator.foldCtx(), from); + long t = foldToLong(toEvaluator.foldCtx(), to); preparedRounding = new DateRoundingPicker(b, f, t).pickRounding().prepareForUnknown(); } else { assert DataType.isTemporalAmount(buckets.dataType()) : "Unexpected span data type [" + buckets.dataType() + "]"; - preparedRounding = DateTrunc.createRounding(buckets.fold(), DEFAULT_TZ); + preparedRounding = DateTrunc.createRounding(buckets.fold(toEvaluator.foldCtx()), DEFAULT_TZ); } return DateTrunc.evaluator(field.dataType(), source(), toEvaluator.apply(field), preparedRounding); } if (field.dataType().isNumeric()) { double roundTo; if (from != null) { - int b = ((Number) buckets.fold()).intValue(); - double f = ((Number) from.fold()).doubleValue(); - double t = ((Number) to.fold()).doubleValue(); + int b = ((Number) buckets.fold(toEvaluator.foldCtx())).intValue(); + double f = ((Number) from.fold(toEvaluator.foldCtx())).doubleValue(); + double t = ((Number) to.fold(toEvaluator.foldCtx())).doubleValue(); roundTo = pickRounding(b, f, t); } else { - roundTo = ((Number) buckets.fold()).doubleValue(); + roundTo = ((Number) buckets.fold(toEvaluator.foldCtx())).doubleValue(); } Literal rounding = new Literal(source(), roundTo, DataType.DOUBLE); @@ -408,7 +409,7 @@ private static TypeResolution isStringOrDate(Expression e, String operationName, } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { String operation = sourceText(); failures.add(isFoldable(buckets, operation, SECOND)) @@ -416,8 +417,8 @@ public void validate(Failures failures) { .add(to != null ? isFoldable(to, operation, FOURTH) : null); } - private long foldToLong(Expression e) { - Object value = Foldables.valueOf(e); + private long foldToLong(FoldContext ctx, Expression e) { + Object value = Foldables.valueOf(ctx, e); return DataType.isDateTime(e.dataType()) ? ((Number) value).longValue() : dateTimeToLong(((BytesRef) value).utf8ToString()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java index a100dd64915f1..570ce7a96dd68 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/Categorize.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; -import org.elasticsearch.xpack.esql.capabilities.Validatable; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -37,7 +36,7 @@ * For the implementation, see {@link org.elasticsearch.compute.aggregation.blockhash.CategorizeBlockHash} *

*/ -public class Categorize extends GroupingFunction implements Validatable { +public class Categorize extends GroupingFunction { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( Expression.class, "Categorize", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java index 0fee65d32ca98..fd025e5e67a7c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/grouping/GroupingFunction.java @@ -10,6 +10,7 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisPlanVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; @@ -28,8 +29,8 @@ protected GroupingFunction(Source source, List fields) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 404ce7e3900c9..85d15f82f458a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.expression.function.scalar; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; @@ -34,7 +35,7 @@ protected EsqlScalarFunction(Source source, List fields) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 824f02ca7ccbb..236e625f7abe1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -227,7 +228,7 @@ public boolean foldable() { if (condition.condition.foldable() == false) { return false; } - if (Boolean.TRUE.equals(condition.condition.fold())) { + if (Boolean.TRUE.equals(condition.condition.fold(FoldContext.small() /* TODO remove me - use literal true?*/))) { /* * `fold` can make four things here: * 1. `TRUE` @@ -264,7 +265,8 @@ public boolean foldable() { * And those two combine so {@code EVAL c=CASE(false, foo, b, bar, true, bort, el)} becomes * {@code EVAL c=CASE(b, bar, bort)}. */ - public Expression partiallyFold() { + public Expression partiallyFold(FoldContext ctx) { + // TODO don't throw away the results of any `fold`. That might mean looking for literal TRUE on the conditions. List newChildren = new ArrayList<>(children().size()); boolean modified = false; for (Condition condition : conditions) { @@ -274,7 +276,7 @@ public Expression partiallyFold() { continue; } modified = true; - if (Boolean.TRUE.equals(condition.condition.fold())) { + if (Boolean.TRUE.equals(condition.condition.fold(ctx))) { /* * `fold` can make four things here: * 1. `TRUE` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java index 8f43a6481db07..57f362f86ff4c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/FoldablesConvertFunction.java @@ -8,9 +8,10 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.convert; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -26,7 +27,7 @@ * Base class for functions that converts a constant into an interval type - DATE_PERIOD or TIME_DURATION. * The functions will be folded at the end of LogicalPlanOptimizer by the coordinator, it does not reach data node. */ -public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements Validatable { +public abstract class FoldablesConvertFunction extends AbstractConvertFunction implements PostOptimizationVerificationAware { protected FoldablesConvertFunction(Source source, Expression field) { super(source, field); @@ -65,12 +66,12 @@ protected final Map factories() { } @Override - public final Object fold() { - return foldToTemporalAmount(field(), sourceText(), dataType()); + public final Object fold(FoldContext ctx) { + return foldToTemporalAmount(ctx, field(), sourceText(), dataType()); } @Override - public final void validate(Failures failures) { + public final void postLogicalOptimizationVerification(Failures failures) { failures.add(isFoldable(field(), sourceText(), null)); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java index 8c4375b424cdc..837475fca7e74 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDateNanos.java @@ -63,9 +63,9 @@ public class ToDateNanos extends AbstractConvertFunction { @FunctionInfo( returnType = "date_nanos", description = "Converts an input to a nanosecond-resolution date value (aka date_nanos).", - note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z. Additionally, integers " - + "cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.", - preview = true + note = "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z, attepting to convert" + + "values outside of that range will result in null with a warning.. Additionally, integers cannot be converted into date " + + "nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch." ) public ToDateNanos( Source source, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index f6a23a5d5962e..b588832aba4cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -232,7 +232,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (unit.foldable()) { try { - Part datePartField = Part.resolve(((BytesRef) unit.fold()).utf8ToString()); + Part datePartField = Part.resolve(((BytesRef) unit.fold(toEvaluator.foldCtx())).utf8ToString()); return new DateDiffConstantEvaluator.Factory(source(), datePartField, startTimestampEvaluator, endTimestampEvaluator); } catch (IllegalArgumentException e) { throw new InvalidArgumentException("invalid unit format for [{}]: {}", sourceText(), e.getMessage()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index 501dfd431f106..7fc5d82441802 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -110,9 +111,9 @@ public String getWriteableName() { public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); if (children().get(0).foldable()) { - ChronoField chrono = chronoField(); + ChronoField chrono = chronoField(toEvaluator.foldCtx()); if (chrono == null) { - BytesRef field = (BytesRef) children().get(0).fold(); + BytesRef field = (BytesRef) children().get(0).fold(toEvaluator.foldCtx()); throw new InvalidArgumentException("invalid date field for [{}]: {}", sourceText(), field.utf8ToString()); } return new DateExtractConstantEvaluator.Factory(source(), fieldEvaluator, chrono, configuration().zoneId()); @@ -121,14 +122,14 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { return new DateExtractEvaluator.Factory(source(), fieldEvaluator, chronoEvaluator, configuration().zoneId()); } - private ChronoField chronoField() { + private ChronoField chronoField(FoldContext ctx) { // chronoField's never checked (the return is). The foldability test is done twice and type is checked in resolveType() already. // TODO: move the slimmed down code here to toEvaluator? if (chronoField == null) { Expression field = children().get(0); try { if (field.foldable() && DataType.isString(field.dataType())) { - chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold()); + chronoField = (ChronoField) STRING_TO_CHRONO_FIELD.convert(field.fold(ctx)); } } catch (Exception e) { return null; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index 60bc014ccbeec..29648d55cadd8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -55,7 +55,7 @@ public class DateFormat extends EsqlConfigurationFunction implements OptionalArg ) public DateFormat( Source source, - @Param(optional = true, name = "dateFormat", type = { "keyword", "text" }, description = """ + @Param(optional = true, name = "dateFormat", type = { "keyword", "text", "date" }, description = """ Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used. If `null`, the function returns `null`.""") Expression format, @Param(name = "date", type = { "date" }, description = "Date expression. If `null`, the function returns `null`.") Expression date, @@ -147,7 +147,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { throw new IllegalArgumentException("unsupported data type for format [" + format.dataType() + "]"); } if (format.foldable()) { - DateFormatter formatter = toFormatter(format.fold(), configuration().locale()); + DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx()), configuration().locale()); return new DateFormatConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } var formatEvaluator = toEvaluator.apply(format); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index e09fabab98d0f..7c38b54ed232b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -143,7 +143,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } if (format.foldable()) { try { - DateFormatter formatter = toFormatter(format.fold()); + DateFormatter formatter = toFormatter(format.fold(toEvaluator.foldCtx())); return new DateParseConstantEvaluator.Factory(source(), fieldEvaluator, formatter); } catch (IllegalArgumentException e) { throw new InvalidArgumentException(e, "invalid date pattern for [{}]: {}", sourceText(), e.getMessage()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index a35b67d7ac3fd..7983c38cc4288 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -225,7 +225,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } Object foldedInterval; try { - foldedInterval = interval.fold(); + foldedInterval = interval.fold(toEvaluator.foldCtx()); if (foldedInterval == null) { throw new IllegalArgumentException("Interval cannot not be null"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index d259fc6ae57ce..74c2da450995c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -59,7 +60,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return now; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java index 757b67b47ce72..e1eceef7ed1f5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/E.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -49,7 +50,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return Math.E; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java index 90a4f1f091e91..32b7a0ab88b4e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Pi.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -49,7 +50,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return Math.PI; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java index 17e5b027270d1..1a7669b7391e1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Tau.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; @@ -51,7 +52,7 @@ public String getWriteableName() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return TAU; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index 1996744a76567..26211258e6ca6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; @@ -91,8 +92,8 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java index 4dd447f938880..d5093964145b7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSum.java @@ -115,7 +115,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua source(), toEvaluator.apply(field), ctx -> new CompensatedSum(), - (Double) p.fold() + (Double) p.fold(toEvaluator.foldCtx()) ); case NULL -> EvalOperator.CONSTANT_NULL_FACTORY; default -> throw EsqlIllegalArgumentException.illegalDataType(field.dataType()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index f4f9679dc3704..4a04524d1b23d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -187,8 +187,8 @@ public boolean foldable() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (start.foldable() && end.foldable()) { - int startOffset = stringToInt(String.valueOf(start.fold())); - int endOffset = stringToInt(String.valueOf(end.fold())); + int startOffset = stringToInt(String.valueOf(start.fold(toEvaluator.foldCtx()))); + int endOffset = stringToInt(String.valueOf(end.fold(toEvaluator.foldCtx()))); checkStartEnd(startOffset, endOffset); } return switch (PlannerUtils.toElementType(field.dataType())) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 2286a1357ced8..b68718acfcd0a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -29,10 +29,11 @@ import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeDouble; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeInt; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupeLong; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -58,7 +59,7 @@ /** * Sorts a multivalued field in lexicographical order. */ -public class MvSort extends EsqlScalarFunction implements OptionalArgument, Validatable { +public class MvSort extends EsqlScalarFunction implements OptionalArgument, PostOptimizationVerificationAware { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSort", MvSort::new); private final Expression field, order; @@ -155,12 +156,12 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua sourceText(), ASC.value(), DESC.value(), - ((BytesRef) order.fold()).utf8ToString() + ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString() ) ); } if (order != null && order.foldable()) { - ordering = ((BytesRef) order.fold()).utf8ToString().equalsIgnoreCase((String) ASC.value()); + ordering = ((BytesRef) order.fold(toEvaluator.foldCtx())).utf8ToString().equalsIgnoreCase((String) ASC.value()); } return switch (PlannerUtils.toElementType(field.dataType())) { @@ -230,7 +231,7 @@ public DataType dataType() { } @Override - public void validate(Failures failures) { + public void postLogicalOptimizationVerification(Failures failures) { if (order == null) { return; } @@ -238,7 +239,14 @@ public void validate(Failures failures) { failures.add(isFoldable(order, operation, SECOND)); if (isValidOrder() == false) { failures.add( - Failure.fail(order, INVALID_ORDER_ERROR, sourceText(), ASC.value(), DESC.value(), ((BytesRef) order.fold()).utf8ToString()) + Failure.fail( + order, + INVALID_ORDER_ERROR, + sourceText(), + ASC.value(), + DESC.value(), + ((BytesRef) order.fold(FoldContext.small() /* TODO remove me */)).utf8ToString() + ) ); } } @@ -246,7 +254,7 @@ public void validate(Failures failures) { private boolean isValidOrder() { boolean isValidOrder = true; if (order != null && order.foldable()) { - Object obj = order.fold(); + Object obj = order.fold(FoldContext.small() /* TODO remove me */); String o = null; if (obj instanceof BytesRef ob) { o = ob.utf8ToString(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java index 9189c6a7b8f70..b15d04aa792d9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialContains.java @@ -26,6 +26,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -214,10 +215,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Geometry rightGeom = makeGeometryFromLiteral(right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Geometry rightGeom = makeGeometryFromLiteral(ctx, right()); Component2D[] components = asLuceneComponent2Ds(crsType(), rightGeom); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometries(docValueReader, components) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java index ee78f50c4d6bd..3e16fa163fcd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialDisjoint.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -129,10 +130,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java index 1a51af8dfeeb4..dcd53075cf69c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialEvaluatorFactory.java @@ -171,7 +171,11 @@ protected static class SpatialEvaluatorWithConstantFactory extends SpatialEvalua @Override public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { - return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2D(s.crsType(), s.right())); + return factoryCreator.apply( + s.source(), + toEvaluator.apply(s.left()), + asLuceneComponent2D(toEvaluator.foldCtx(), s.crsType(), s.right()) + ); } } @@ -197,7 +201,11 @@ protected static class SpatialEvaluatorWithConstantArrayFactory extends SpatialE @Override public EvalOperator.ExpressionEvaluator.Factory get(SpatialSourceSupplier s, EvaluatorMapper.ToEvaluator toEvaluator) { - return factoryCreator.apply(s.source(), toEvaluator.apply(s.left()), asLuceneComponent2Ds(s.crsType(), s.right())); + return factoryCreator.apply( + s.source(), + toEvaluator.apply(s.left()), + asLuceneComponent2Ds(toEvaluator.foldCtx(), s.crsType(), s.right()) + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java index 8d54e5ee443c2..601550cd173bb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialIntersects.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -127,10 +128,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java index 6ae99ea8165cd..1b06c6dfd3dd5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java @@ -29,6 +29,7 @@ import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.lucene.spatial.GeometryDocValueWriter; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -42,8 +43,8 @@ public class SpatialRelatesUtils { /** Converts a {@link Expression} into a {@link Component2D}. */ - static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { - return asLuceneComponent2D(crsType, makeGeometryFromLiteral(expression)); + static Component2D asLuceneComponent2D(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { + return asLuceneComponent2D(crsType, makeGeometryFromLiteral(ctx, expression)); } /** Converts a {@link Geometry} into a {@link Component2D}. */ @@ -66,8 +67,8 @@ static Component2D asLuceneComponent2D(BinarySpatialFunction.SpatialCrsType type * Converts a {@link Expression} at a given {@code position} into a {@link Component2D} array. * The reason for generating an array instead of a single component is for multi-shape support with ST_CONTAINS. */ - static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { - return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(expression)); + static Component2D[] asLuceneComponent2Ds(FoldContext ctx, BinarySpatialFunction.SpatialCrsType crsType, Expression expression) { + return asLuceneComponent2Ds(crsType, makeGeometryFromLiteral(ctx, expression)); } /** @@ -90,9 +91,12 @@ static Component2D[] asLuceneComponent2Ds(BinarySpatialFunction.SpatialCrsType t } /** Converts a {@link Expression} into a {@link GeometryDocValueReader} */ - static GeometryDocValueReader asGeometryDocValueReader(BinarySpatialFunction.SpatialCrsType crsType, Expression expression) - throws IOException { - Geometry geometry = makeGeometryFromLiteral(expression); + static GeometryDocValueReader asGeometryDocValueReader( + FoldContext ctx, + BinarySpatialFunction.SpatialCrsType crsType, + Expression expression + ) throws IOException { + Geometry geometry = makeGeometryFromLiteral(ctx, expression); if (crsType == BinarySpatialFunction.SpatialCrsType.GEO) { return asGeometryDocValueReader( CoordinateEncoder.GEO, @@ -167,8 +171,8 @@ private static Geometry asGeometry(BytesRefBlock valueBlock, int position) { * This function is used in two places, when evaluating a spatial constant in the SpatialRelatesFunction, as well as when * we do lucene-pushdown of spatial functions. */ - public static Geometry makeGeometryFromLiteral(Expression expr) { - return makeGeometryFromLiteralValue(valueOf(expr), expr.dataType()); + public static Geometry makeGeometryFromLiteral(FoldContext ctx, Expression expr) { + return makeGeometryFromLiteralValue(valueOf(ctx, expr), expr.dataType()); } private static Geometry makeGeometryFromLiteralValue(Object value, DataType dataType) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java index 2005709cd37e9..9fcece1ce65bc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialWithin.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.lucene.spatial.GeometryDocValueReader; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -129,10 +130,10 @@ protected NodeInfo info() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { try { - GeometryDocValueReader docValueReader = asGeometryDocValueReader(crsType(), left()); - Component2D component2D = asLuceneComponent2D(crsType(), right()); + GeometryDocValueReader docValueReader = asGeometryDocValueReader(ctx, crsType(), left()); + Component2D component2D = asLuceneComponent2D(ctx, crsType(), right()); return (crsType() == SpatialCrsType.GEO) ? GEO.geometryRelatesGeometry(docValueReader, component2D) : CARTESIAN.geometryRelatesGeometry(docValueReader, component2D); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java index 3cf042a2db828..f0c25e3289cc1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/StDistance.java @@ -23,6 +23,7 @@ import org.elasticsearch.lucene.spatial.CoordinateEncoder; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -280,18 +281,18 @@ protected NodeInfo info() { } @Override - public Object fold() { - var leftGeom = makeGeometryFromLiteral(left()); - var rightGeom = makeGeometryFromLiteral(right()); + public Object fold(FoldContext ctx) { + var leftGeom = makeGeometryFromLiteral(ctx, left()); + var rightGeom = makeGeometryFromLiteral(ctx, right()); return (crsType() == SpatialCrsType.GEO) ? GEO.distance(leftGeom, rightGeom) : CARTESIAN.distance(leftGeom, rightGeom); } @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable()) { - return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(right()), leftDocValues); + return toEvaluator(toEvaluator, left(), makeGeometryFromLiteral(toEvaluator.foldCtx(), right()), leftDocValues); } else if (left().foldable()) { - return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(left()), rightDocValues); + return toEvaluator(toEvaluator, right(), makeGeometryFromLiteral(toEvaluator.foldCtx(), left()), rightDocValues); } else { EvalOperator.ExpressionEvaluator.Factory leftE = toEvaluator.apply(left()); EvalOperator.ExpressionEvaluator.Factory rightE = toEvaluator.apply(right()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java index 52d33c0fc9d3d..be0a7b2fe27b2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Hash.java @@ -146,7 +146,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua if (algorithm.foldable()) { try { // hash function is created here in order to validate the algorithm is valid before evaluator is created - var hf = HashFunction.create((BytesRef) algorithm.fold()); + var hf = HashFunction.create((BytesRef) algorithm.fold(toEvaluator.foldCtx())); return new HashConstantEvaluator.Factory( source(), context -> new BreakingBytesRefBuilder(context.breaker(), "hash"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java index 996c90a8e40bc..fb0aac0c85b38 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLike.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -94,8 +95,8 @@ protected TypeResolution resolveType() { } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java index e91f03de3dd7e..363991d1556f1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Repeat.java @@ -151,7 +151,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { ExpressionEvaluator.Factory strExpr = toEvaluator.apply(str); if (number.foldable()) { - int num = (int) number.fold(); + int num = (int) number.fold(toEvaluator.foldCtx()); if (num < 0) { throw new IllegalArgumentException("Number parameter cannot be negative, found [" + number + "]"); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java index 4fa191244cb42..4b963b794aef0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Replace.java @@ -152,7 +152,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (regex.foldable() && regex.dataType() == DataType.KEYWORD) { Pattern regexPattern; try { - regexPattern = Pattern.compile(((BytesRef) regex.fold()).utf8ToString()); + regexPattern = Pattern.compile(((BytesRef) regex.fold(toEvaluator.foldCtx())).utf8ToString()); } catch (PatternSyntaxException pse) { // TODO this is not right (inconsistent). See also https://github.com/elastic/elasticsearch/issues/100038 // this should generate a header warning and return null (as do the rest of this functionality in evaluators), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java index 3b9a466966911..e46c0a730431d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Space.java @@ -113,7 +113,7 @@ protected NodeInfo info() { @Override public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (field.foldable()) { - Object folded = field.fold(); + Object folded = field.fold(toEvaluator.foldCtx()); if (folded instanceof Integer num) { checkNumber(num); return toEvaluator.apply(new Literal(source(), " ".repeat(num), KEYWORD)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java index 24762122f755b..d0c1035978ff3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Split.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -102,8 +103,8 @@ public boolean foldable() { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } @Evaluator(extraName = "SingleByte") @@ -163,7 +164,7 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { if (right().foldable() == false) { return new SplitVariableEvaluator.Factory(source(), str, toEvaluator.apply(right()), context -> new BytesRef()); } - BytesRef delim = (BytesRef) right().fold(); + BytesRef delim = (BytesRef) right().fold(toEvaluator.foldCtx()); checkDelimiter(delim); return new SplitSingleByteEvaluator.Factory(source(), str, delim.bytes[delim.offset], context -> new BytesRef()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java index d2edb0f92e8f2..65455c708cc9b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLike.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -100,8 +101,8 @@ protected TypeResolution resolveType() { } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java index 1d03f09c86409..3b17133bf4974 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/util/Delay.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -84,15 +85,15 @@ public boolean foldable() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { return null; } - private long msValue() { + private long msValue(FoldContext ctx) { if (field().foldable() == false) { throw new IllegalArgumentException("function [" + sourceText() + "] has invalid argument [" + field().sourceText() + "]"); } - var ms = field().fold(); + var ms = field().fold(ctx); if (ms instanceof Duration duration) { return duration.toMillis(); } @@ -101,7 +102,7 @@ private long msValue() { @Override public ExpressionEvaluator.Factory toEvaluator(EvaluatorMapper.ToEvaluator toEvaluator) { - return new DelayEvaluator.Factory(source(), msValue()); + return new DelayEvaluator.Factory(source(), msValue(toEvaluator.foldCtx())); } @Evaluator diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 8bb166fac60bb..424c080c905e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -111,7 +112,7 @@ protected TypeResolution checkCompatibility() { /** * Override this to allow processing literals of type {@link DataType#DATE_PERIOD} when folding constants. - * Used in {@link DateTimeArithmeticOperation#fold()}. + * Used in {@link DateTimeArithmeticOperation#fold}. * @param left the left period * @param right the right period * @return the result of the evaluation @@ -120,7 +121,7 @@ protected TypeResolution checkCompatibility() { /** * Override this to allow processing literals of type {@link DataType#TIME_DURATION} when folding constants. - * Used in {@link DateTimeArithmeticOperation#fold()}. + * Used in {@link DateTimeArithmeticOperation#fold}. * @param left the left duration * @param right the right duration * @return the result of the evaluation @@ -128,13 +129,13 @@ protected TypeResolution checkCompatibility() { abstract Duration fold(Duration left, Duration right); @Override - public final Object fold() { + public final Object fold(FoldContext ctx) { DataType leftDataType = left().dataType(); DataType rightDataType = right().dataType(); if (leftDataType == DATE_PERIOD && rightDataType == DATE_PERIOD) { // Both left and right expressions are temporal amounts; we can assume they are both foldable. - var l = left().fold(); - var r = right().fold(); + var l = left().fold(ctx); + var r = right().fold(ctx); if (l instanceof Collection || r instanceof Collection) { return null; } @@ -148,8 +149,8 @@ public final Object fold() { } if (leftDataType == TIME_DURATION && rightDataType == TIME_DURATION) { // Both left and right expressions are temporal amounts; we can assume they are both foldable. - Duration l = (Duration) left().fold(); - Duration r = (Duration) right().fold(); + Duration l = (Duration) left().fold(ctx); + Duration r = (Duration) right().fold(ctx); try { return fold(l, r); } catch (ArithmeticException e) { @@ -161,7 +162,7 @@ public final Object fold() { if (isNull(leftDataType) || isNull(rightDataType)) { return null; } - return super.fold(); + return super.fold(ctx); } @Override @@ -178,7 +179,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { temporalAmountArgument = left(); } - return millisEvaluator.apply(source(), toEvaluator.apply(datetimeArgument), (TemporalAmount) temporalAmountArgument.fold()); + return millisEvaluator.apply( + source(), + toEvaluator.apply(datetimeArgument), + (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx()) + ); } else if (dataType() == DATE_NANOS) { // One of the arguments has to be a date_nanos and the other a temporal amount. Expression dateNanosArgument; @@ -191,7 +196,11 @@ public ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { temporalAmountArgument = left(); } - return nanosEvaluator.apply(source(), toEvaluator.apply(dateNanosArgument), (TemporalAmount) temporalAmountArgument.fold()); + return nanosEvaluator.apply( + source(), + toEvaluator.apply(dateNanosArgument), + (TemporalAmount) temporalAmountArgument.fold(toEvaluator.foldCtx()) + ); } else { return super.toEvaluator(toEvaluator); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 74394d796855f..e3248665ad486 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryArithmeticOperation; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -120,8 +121,8 @@ public interface BinaryEvaluator { } @Override - public Object fold() { - return EvaluatorMapper.super.fold(); + public Object fold(FoldContext ctx) { + return EvaluatorMapper.super.fold(source(), ctx); } public DataType dataType() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java index fb32282005f02..6663ccf0ef7b6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Neg.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -87,12 +88,12 @@ else if (type == DataType.LONG) { } @Override - public final Object fold() { + public final Object fold(FoldContext ctx) { DataType dataType = field().dataType(); // For date periods and time durations, we need to treat folding differently. These types are unrepresentable, so there is no // evaluator for them - but the default folding requires an evaluator. if (dataType == DATE_PERIOD) { - Period fieldValue = (Period) field().fold(); + Period fieldValue = (Period) field().fold(ctx); try { return fieldValue.negated(); } catch (ArithmeticException e) { @@ -102,7 +103,7 @@ public final Object fold() { } } if (dataType == TIME_DURATION) { - Duration fieldValue = (Duration) field().fold(); + Duration fieldValue = (Duration) field().fold(ctx); try { return fieldValue.negated(); } catch (ArithmeticException e) { @@ -111,7 +112,7 @@ public final Object fold() { throw ExceptionUtils.math(source(), e); } } - return super.fold(); + return super.fold(ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 3e2a21664aa7e..e56c19b26a902 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -204,8 +205,8 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua } @Override - public Boolean fold() { - return (Boolean) EvaluatorMapper.super.fold(); + public Boolean fold(FoldContext ctx) { + return (Boolean) EvaluatorMapper.super.fold(source(), ctx); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java index f596d589cdde2..6a35fa8e8b957 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/In.java @@ -11,12 +11,14 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Vector; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.Comparisons; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -40,6 +42,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; @@ -205,11 +208,11 @@ public boolean foldable() { } @Override - public Object fold() { + public Object fold(FoldContext ctx) { if (Expressions.isGuaranteedNull(value) || list.stream().allMatch(Expressions::isGuaranteedNull)) { return null; } - return super.fold(); + return super.fold(ctx); } protected boolean areCompatible(DataType left, DataType right) { @@ -231,20 +234,47 @@ protected TypeResolution resolveType() { // TODO: move the foldability check fro } DataType dt = value.dataType(); - for (int i = 0; i < list.size(); i++) { - Expression listValue = list.get(i); - if (areCompatible(dt, listValue.dataType()) == false) { - return new TypeResolution( - format( - null, - "{} argument of [{}] must be [{}], found value [{}] type [{}]", - ordinal(i + 1), - sourceText(), - dt.typeName(), - Expressions.name(listValue), - listValue.dataType().typeName() - ) - ); + if (dt.isDate()) { + // If value is a date (nanos or millis), list cannot contain both nanos and millis + DataType seenDateType = null; + for (int i = 0; i < list.size(); i++) { + Expression listValue = list.get(i); + if (seenDateType == null && listValue.dataType().isDate()) { + seenDateType = listValue.dataType(); + } + // The areCompatible test is still necessary to account for nulls. + if ((listValue.dataType().isDate() && listValue.dataType() != seenDateType) + || (listValue.dataType().isDate() == false && areCompatible(dt, listValue.dataType()) == false)) { + return new TypeResolution( + format( + null, + "{} argument of [{}] must be [{}], found value [{}] type [{}]", + ordinal(i + 1), + sourceText(), + dt.typeName(), + Expressions.name(listValue), + listValue.dataType().typeName() + ) + ); + } + } + + } else { + for (int i = 0; i < list.size(); i++) { + Expression listValue = list.get(i); + if (areCompatible(dt, listValue.dataType()) == false) { + return new TypeResolution( + format( + null, + "{} argument of [{}] must be [{}], found value [{}] type [{}]", + ordinal(i + 1), + sourceText(), + dt.typeName(), + Expressions.name(listValue), + listValue.dataType().typeName() + ) + ); + } } } @@ -261,9 +291,19 @@ protected Expression canonicalize() { @Override public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { - var commonType = commonType(); EvalOperator.ExpressionEvaluator.Factory lhs; EvalOperator.ExpressionEvaluator.Factory[] factories; + if (value.dataType() == DATE_NANOS && list.getFirst().dataType() == DATETIME) { + lhs = toEvaluator.apply(value); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + return new InNanosMillisEvaluator.Factory(source(), lhs, factories); + } + if (value.dataType() == DATETIME && list.getFirst().dataType() == DATE_NANOS) { + lhs = toEvaluator.apply(value); + factories = list.stream().map(toEvaluator::apply).toArray(EvalOperator.ExpressionEvaluator.Factory[]::new); + return new InMillisNanosEvaluator.Factory(source(), lhs, factories); + } + var commonType = commonType(); if (commonType.isNumeric()) { lhs = Cast.cast(source(), value.dataType(), commonType, toEvaluator.apply(value)); factories = list.stream() @@ -283,7 +323,7 @@ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvalua if (commonType == INTEGER) { return new InIntEvaluator.Factory(source(), lhs, factories); } - if (commonType == LONG || commonType == DATETIME || commonType == UNSIGNED_LONG) { + if (commonType == LONG || commonType == DATETIME || commonType == DATE_NANOS || commonType == UNSIGNED_LONG) { return new InLongEvaluator.Factory(source(), lhs, factories); } if (commonType == KEYWORD @@ -345,6 +385,39 @@ static boolean process(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { return false; } + /** + * Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in nanoseconds + * and the "list" (aka rhs) is in milliseconds + */ + static boolean processNanosMillis(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + if (DateUtils.compareNanosToMillis(lhs, rhs[i]) == 0) { + return true; + } + } + return false; + } + + /** + * Processor for mixed millisecond and nanosecond dates, where the "value" (aka lhs) is in milliseoncds + * and the "list" (aka rhs) is in nanoseconds + */ + static boolean processMillisNanos(BitSet nulls, BitSet mvs, long lhs, long[] rhs) { + for (int i = 0; i < rhs.length; i++) { + if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { + continue; + } + Boolean compResult = DateUtils.compareNanosToMillis(rhs[i], lhs) == 0; + if (compResult == Boolean.TRUE) { + return true; + } + } + return false; + } + static boolean process(BitSet nulls, BitSet mvs, double lhs, double[] rhs) { for (int i = 0; i < rhs.length; i++) { if ((nulls != null && nulls.get(i)) || (mvs != null && mvs.get(i))) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java index c731e44197f2e..01564644bf5c7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -77,9 +78,9 @@ public static Automaton automaton(BytesRef val) { } @Override - public Boolean fold() { - BytesRef leftVal = BytesRefs.toBytesRef(left().fold()); - BytesRef rightVal = BytesRefs.toBytesRef(right().fold()); + public Boolean fold(FoldContext ctx) { + BytesRef leftVal = BytesRefs.toBytesRef(left().fold(ctx)); + BytesRef rightVal = BytesRefs.toBytesRef(right().fold(ctx)); if (leftVal == null || rightVal == null) { return null; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java index f5704239993f9..7ea95c764f36c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsMapper.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.ExpressionMapper; @@ -28,15 +29,15 @@ public class InsensitiveEqualsMapper extends ExpressionMapper InsensitiveEqualsEvaluator.Factory::new; @Override - public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout) { + public final ExpressionEvaluator.Factory map(FoldContext foldCtx, InsensitiveEquals bc, Layout layout) { DataType leftType = bc.left().dataType(); DataType rightType = bc.right().dataType(); - var leftEval = toEvaluator(bc.left(), layout); - var rightEval = toEvaluator(bc.right(), layout); + var leftEval = toEvaluator(foldCtx, bc.left(), layout); + var rightEval = toEvaluator(foldCtx, bc.right(), layout); if (DataType.isString(leftType)) { if (bc.right().foldable() && DataType.isString(rightType)) { - BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold()); + BytesRef rightVal = BytesRefs.toBytesRef(bc.right().fold(FoldContext.small() /* TODO remove me */)); Automaton automaton = InsensitiveEquals.automaton(rightVal); return dvrCtx -> new InsensitiveEqualsConstantEvaluator( bc.source(), @@ -51,13 +52,14 @@ public final ExpressionEvaluator.Factory map(InsensitiveEquals bc, Layout layout } public static ExpressionEvaluator.Factory castToEvaluator( + FoldContext foldCtx, InsensitiveEquals op, Layout layout, DataType required, TriFunction factory ) { - var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(op.left(), layout)); - var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(op.right(), layout)); + var lhs = Cast.cast(op.source(), op.left().dataType(), required, toEvaluator(foldCtx, op.left(), layout)); + var rhs = Cast.cast(op.source(), op.right().dataType(), required, toEvaluator(foldCtx, op.right(), layout)); return factory.apply(op.source(), lhs, rhs); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st index f63815f475d8f..0cdea82119cae 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st @@ -42,7 +42,7 @@ import java.util.BitSet; * {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}. * This class is generated. Edit {@code X-InEvaluator.java.st} instead. */ -public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator { +public class In$Name$Evaluator implements EvalOperator.ExpressionEvaluator { private final Source source; private final EvalOperator.ExpressionEvaluator lhs; @@ -53,7 +53,7 @@ public class In$Type$Evaluator implements EvalOperator.ExpressionEvaluator { private Warnings warnings; - public In$Type$Evaluator( + public In$Name$Evaluator( Source source, EvalOperator.ExpressionEvaluator lhs, EvalOperator.ExpressionEvaluator[] rhs, @@ -166,6 +166,9 @@ $if(boolean)$ foundMatch = lhsBlock.getBoolean(lhsBlock.getFirstValueIndex(p)) ? hasTrue : hasFalse; $elseif(BytesRef)$ foundMatch = In.process(nulls, mvs, lhsBlock.getBytesRef(lhsBlock.getFirstValueIndex(p), lhsScratch), rhsValues); +$elseif(NanosMillis)$ + // Even though these are longs, they're in different units and need special handling + foundMatch = In.processNanosMilils(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues); $else$ foundMatch = In.process(nulls, mvs, lhsBlock.get$Type$(lhsBlock.getFirstValueIndex(p)), rhsValues); $endif$ @@ -230,6 +233,10 @@ $if(BytesRef)$ result.appendBoolean(In.process(null, null, lhsVector.getBytesRef(p, lhsScratch), rhsValues)); $elseif(boolean)$ result.appendBoolean(lhsVector.getBoolean(p) ? hasTrue : hasFalse); +$elseif(nanosMillis)$ + result.appendBoolean(In.processNanosMillis(null, null, lhsVector.get$Type$(p), rhsValues)); +$elseif(millisNanos)$ + result.appendBoolean(In.processMillisNanos(null, null, lhsVector.get$Type$(p), rhsValues)); $else$ result.appendBoolean(In.process(null, null, lhsVector.get$Type$(p), rhsValues)); $endif$ @@ -240,7 +247,7 @@ $endif$ @Override public String toString() { - return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; } @Override @@ -272,16 +279,16 @@ $endif$ } @Override - public In$Type$Evaluator get(DriverContext context) { + public In$Name$Evaluator get(DriverContext context) { EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs) .map(a -> a.get(context)) .toArray(EvalOperator.ExpressionEvaluator[]::new); - return new In$Type$Evaluator(source, lhs.get(context), rhs, context); + return new In$Name$Evaluator(source, lhs.get(context), rhs, context); } @Override public String toString() { - return "In$Type$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; + return "In$Name$Evaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]"; } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java index ef5cf50c76541..183008f900c5d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalOptimizerContext.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; @@ -15,8 +16,8 @@ public final class LocalLogicalOptimizerContext extends LogicalOptimizerContext { private final SearchStats searchStats; - public LocalLogicalOptimizerContext(Configuration configuration, SearchStats searchStats) { - super(configuration); + public LocalLogicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) { + super(configuration, foldCtx); this.searchStats = searchStats; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java index c11e1a4ec49e4..22e07b45310fb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalOptimizerContext.java @@ -7,7 +7,8 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.stats.SearchStats; -public record LocalPhysicalOptimizerContext(Configuration configuration, SearchStats searchStats) {} +public record LocalPhysicalOptimizerContext(Configuration configuration, FoldContext foldCtx, SearchStats searchStats) {} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java index 67148e67cbc19..da2d583674a90 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalOptimizerContext.java @@ -7,37 +7,44 @@ package org.elasticsearch.xpack.esql.optimizer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.session.Configuration; import java.util.Objects; public class LogicalOptimizerContext { private final Configuration configuration; + private final FoldContext foldCtx; - public LogicalOptimizerContext(Configuration configuration) { + public LogicalOptimizerContext(Configuration configuration, FoldContext foldCtx) { this.configuration = configuration; + this.foldCtx = foldCtx; } public Configuration configuration() { return configuration; } + public FoldContext foldCtx() { + return foldCtx; + } + @Override public boolean equals(Object obj) { if (obj == this) return true; if (obj == null || obj.getClass() != this.getClass()) return false; var that = (LogicalOptimizerContext) obj; - return Objects.equals(this.configuration, that.configuration); + return this.configuration.equals(that.configuration) && this.foldCtx.equals(that.foldCtx); } @Override public int hashCode() { - return Objects.hash(configuration); + return Objects.hash(configuration, foldCtx); } @Override public String toString() { - return "LogicalOptimizerContext[" + "configuration=" + configuration + ']'; + return "LogicalOptimizerContext[configuration=" + configuration + ", foldCtx=" + foldCtx + ']'; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java index dce828dbf192d..7e6eddd3ef041 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalVerifier.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.esql.optimizer; -import org.elasticsearch.xpack.esql.capabilities.Validatable; +import org.elasticsearch.xpack.esql.capabilities.PostOptimizationVerificationAware; import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.optimizer.rules.PlanConsistencyChecker; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -28,8 +28,8 @@ public Failures verify(LogicalPlan plan) { if (failures.hasFailures() == false) { p.forEachExpression(ex -> { - if (ex instanceof Validatable v) { - v.validate(failures); + if (ex instanceof PostOptimizationVerificationAware va) { + va.postLogicalOptimizationVerification(failures); } }); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java index 3152f9b574767..5f463f2aa4c78 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsElimination.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; @@ -28,7 +29,7 @@ public BooleanFunctionEqualsElimination() { } @Override - public Expression rule(BinaryComparison bc) { + public Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) { if ((bc instanceof Equals || bc instanceof NotEquals) && bc.left() instanceof Function) { // for expression "==" or "!=" TRUE/FALSE, return the expression itself or its negated variant diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java index 73d1ea1fb6e8f..e1803872fd606 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplification.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.List; @@ -35,7 +36,7 @@ public BooleanSimplification() { } @Override - public Expression rule(ScalarFunction e) { + public Expression rule(ScalarFunction e, LogicalOptimizerContext ctx) { if (e instanceof And || e instanceof Or) { return simplifyAndOr((BinaryPredicate) e); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java index 3f47c74aaf814..1c290a7c4c4fd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisons.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -31,17 +33,17 @@ public CombineBinaryComparisons() { } @Override - public Expression rule(BinaryLogic e) { + public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) { if (e instanceof And and) { - return combine(and); + return combine(ctx.foldCtx(), and); } else if (e instanceof Or or) { - return combine(or); + return combine(ctx.foldCtx(), or); } return e; } // combine conjunction - private static Expression combine(And and) { + private static Expression combine(FoldContext ctx, And and) { List bcs = new ArrayList<>(); List exps = new ArrayList<>(); boolean changed = false; @@ -58,13 +60,13 @@ private static Expression combine(And and) { }); for (Expression ex : andExps) { if (ex instanceof BinaryComparison bc && (ex instanceof Equals || ex instanceof NotEquals) == false) { - if (bc.right().foldable() && (findExistingComparison(bc, bcs, true))) { + if (bc.right().foldable() && (findExistingComparison(ctx, bc, bcs, true))) { changed = true; } else { bcs.add(bc); } } else if (ex instanceof NotEquals neq) { - if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(neq, bcs)) { + if (neq.right().foldable() && notEqualsIsRemovableFromConjunction(ctx, neq, bcs)) { // the non-equality can simply be dropped: either superfluous or has been merged with an updated range/inequality changed = true; } else { // not foldable OR not overlapping @@ -78,13 +80,13 @@ private static Expression combine(And and) { } // combine disjunction - private static Expression combine(Or or) { + private static Expression combine(FoldContext ctx, Or or) { List bcs = new ArrayList<>(); List exps = new ArrayList<>(); boolean changed = false; for (Expression ex : Predicates.splitOr(or)) { if (ex instanceof BinaryComparison bc) { - if (bc.right().foldable() && findExistingComparison(bc, bcs, false)) { + if (bc.right().foldable() && findExistingComparison(ctx, bc, bcs, false)) { changed = true; } else { bcs.add(bc); @@ -100,8 +102,8 @@ private static Expression combine(Or or) { * Find commonalities between the given comparison in the given list. * The method can be applied both for conjunctive (AND) or disjunctive purposes (OR). */ - private static boolean findExistingComparison(BinaryComparison main, List bcs, boolean conjunctive) { - Object value = main.right().fold(); + private static boolean findExistingComparison(FoldContext ctx, BinaryComparison main, List bcs, boolean conjunctive) { + Object value = main.right().fold(ctx); // NB: the loop modifies the list (hence why the int is used) for (int i = 0; i < bcs.size(); i++) { BinaryComparison other = bcs.get(i); @@ -113,7 +115,7 @@ private static boolean findExistingComparison(BinaryComparison main, List bcs) { - Object neqVal = notEquals.right().fold(); + private static boolean notEqualsIsRemovableFromConjunction(FoldContext ctx, NotEquals notEquals, List bcs) { + Object neqVal = notEquals.right().fold(ctx); Integer comp; // check on "condition-overlapping" inequalities: @@ -183,7 +185,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, BinaryComparison bc = bcs.get(i); if (notEquals.left().semanticEquals(bc.left())) { if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { - comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null; + comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null; if (comp != null) { if (comp >= 0) { if (comp == 0 && bc instanceof LessThanOrEqual) { // a != 2 AND a <= 2 -> a < 2 @@ -193,7 +195,7 @@ private static boolean notEqualsIsRemovableFromConjunction(NotEquals notEquals, } // else: comp < 0 : a != 2 AND a nop } // else: non-comparable, nop } else if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { - comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold()) : null; + comp = bc.right().foldable() ? BinaryComparison.compare(neqVal, bc.right().fold(ctx)) : null; if (comp != null) { if (comp <= 0) { if (comp == 0 && bc instanceof GreaterThanOrEqual) { // a != 2 AND a >= 2 -> a > 2 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java index 5cb377de47efc..e1cda9cb149d4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctions.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.time.ZoneId; import java.util.ArrayList; @@ -61,7 +62,7 @@ protected static CIDRMatch createCIDRMatch(Expression k, List v) { } @Override - public Expression rule(Or or) { + public Expression rule(Or or, LogicalOptimizerContext ctx) { Expression e = or; // look only at equals, In and CIDRMatch List exps = splitOr(e); @@ -78,7 +79,7 @@ public Expression rule(Or or) { if (eq.right().foldable()) { ins.computeIfAbsent(eq.left(), k -> new LinkedHashSet<>()).add(eq.right()); if (eq.left().dataType() == DataType.IP) { - Object value = eq.right().fold(); + Object value = eq.right().fold(ctx.foldCtx()); // ImplicitCasting and ConstantFolding(includes explicit casting) are applied before CombineDisjunctions. // They fold the input IP string to an internal IP format. These happen to Equals and IN, but not for CIDRMatch, // as CIDRMatch takes strings as input, ImplicitCasting does not apply to it, and the first input to CIDRMatch is a @@ -101,7 +102,7 @@ public Expression rule(Or or) { if (in.value().dataType() == DataType.IP) { List values = new ArrayList<>(in.list().size()); for (Expression i : in.list()) { - Object value = i.fold(); + Object value = i.fold(ctx.foldCtx()); // Same as Equals. if (value instanceof BytesRef bytesRef) { value = ipToString(bytesRef); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java index 82fe2c6bddf50..27eec8de59020 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFolding.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public final class ConstantFolding extends OptimizerRules.OptimizerExpressionRule { @@ -17,7 +18,7 @@ public ConstantFolding() { } @Override - public Expression rule(Expression e) { - return e.foldable() ? Literal.of(e) : e; + public Expression rule(Expression e, LogicalOptimizerContext ctx) { + return e.foldable() ? Literal.of(ctx.foldCtx(), e) : e; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java index 0604750883f14..b716d8f012d21 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConvertStringToByteRef.java @@ -10,6 +10,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -21,7 +22,8 @@ public ConvertStringToByteRef() { } @Override - protected Expression rule(Literal lit) { + protected Expression rule(Literal lit, LogicalOptimizerContext ctx) { + // TODO we shouldn't be emitting String into Literals at all Object value = lit.value(); if (value == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java index 747864625e65c..cf4c7f19baafe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNull.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public class FoldNull extends OptimizerRules.OptimizerExpressionRule { @@ -23,7 +24,7 @@ public FoldNull() { } @Override - public Expression rule(Expression e) { + public Expression rule(Expression e, LogicalOptimizerContext ctx) { Expression result = tryReplaceIsNullIsNotNull(e); // convert an aggregate null filter into a false diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java index d96c73d5ee4f1..6504e6042c33a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRight.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public final class LiteralsOnTheRight extends OptimizerRules.OptimizerExpressionRule> { @@ -17,7 +18,7 @@ public LiteralsOnTheRight() { } @Override - public BinaryOperator rule(BinaryOperator be) { + public BinaryOperator rule(BinaryOperator be, LogicalOptimizerContext ctx) { return be.left() instanceof Literal && (be.right() instanceof Literal) == false ? be.swapLeftAndRight() : be; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java index 2a0b2a6af36aa..169ac2ac8c0fe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/OptimizerRules.java @@ -8,6 +8,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.util.ReflectionUtils; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.rule.ParameterizedRule; import org.elasticsearch.xpack.esql.rule.Rule; @@ -36,7 +37,10 @@ public final LogicalPlan apply(LogicalPlan plan) { protected abstract LogicalPlan rule(SubPlan plan); } - public abstract static class OptimizerExpressionRule extends Rule { + public abstract static class OptimizerExpressionRule extends ParameterizedRule< + LogicalPlan, + LogicalPlan, + LogicalOptimizerContext> { private final TransformDirection direction; // overriding type token which returns the correct class but does an uncheck cast to LogicalPlan due to its generic bound @@ -49,17 +53,13 @@ public OptimizerExpressionRule(TransformDirection direction) { } @Override - public final LogicalPlan apply(LogicalPlan plan) { + public final LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) { return direction == TransformDirection.DOWN - ? plan.transformExpressionsDown(expressionTypeToken, this::rule) - : plan.transformExpressionsUp(expressionTypeToken, this::rule); + ? plan.transformExpressionsDown(expressionTypeToken, e -> rule(e, ctx)) + : plan.transformExpressionsUp(expressionTypeToken, e -> rule(e, ctx)); } - protected LogicalPlan rule(LogicalPlan plan) { - return plan; - } - - protected abstract Expression rule(E e); + protected abstract Expression rule(E e, LogicalOptimizerContext ctx); public Class expressionToken() { return expressionTypeToken; @@ -82,6 +82,7 @@ protected ParameterizedOptimizerRule(TransformDirection direction) { this.direction = direction; } + @Override public final LogicalPlan apply(LogicalPlan plan, P context) { return direction == TransformDirection.DOWN ? plan.transformDown(typeToken(), t -> rule(t, context)) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java index 118e4fc170520..0111c7cdd806a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PartiallyFoldCase.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import static org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection.DOWN; @@ -28,7 +29,7 @@ public PartiallyFoldCase() { } @Override - protected Expression rule(Case c) { - return c.partiallyFold(); + protected Expression rule(Case c, LogicalOptimizerContext ctx) { + return c.partiallyFold(ctx.foldCtx()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java index 8437b79454884..b6f185c856693 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEmptyRelation.java @@ -12,9 +12,11 @@ import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; @@ -26,15 +28,18 @@ import java.util.List; @SuppressWarnings("removal") -public class PropagateEmptyRelation extends OptimizerRules.OptimizerRule { +public class PropagateEmptyRelation extends OptimizerRules.ParameterizedOptimizerRule { + public PropagateEmptyRelation() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - protected LogicalPlan rule(UnaryPlan plan) { + protected LogicalPlan rule(UnaryPlan plan, LogicalOptimizerContext ctx) { LogicalPlan p = plan; if (plan.child() instanceof LocalRelation local && local.supplier() == LocalSupplier.EMPTY) { // only care about non-grouped aggs might return something (count) if (plan instanceof Aggregate agg && agg.groupings().isEmpty()) { - List emptyBlocks = aggsFromEmpty(agg.aggregates()); + List emptyBlocks = aggsFromEmpty(ctx.foldCtx(), agg.aggregates()); p = replacePlanByRelation(plan, LocalSupplier.of(emptyBlocks.toArray(Block[]::new))); } else { p = PruneEmptyPlans.skipPlan(plan); @@ -43,14 +48,14 @@ protected LogicalPlan rule(UnaryPlan plan) { return p; } - private List aggsFromEmpty(List aggs) { + private List aggsFromEmpty(FoldContext foldCtx, List aggs) { List blocks = new ArrayList<>(); var blockFactory = PlannerUtils.NON_BREAKING_BLOCK_FACTORY; int i = 0; for (var agg : aggs) { // there needs to be an alias if (Alias.unwrap(agg) instanceof AggregateFunction aggFunc) { - aggOutput(agg, aggFunc, blockFactory, blocks); + aggOutput(foldCtx, agg, aggFunc, blockFactory, blocks); } else { throw new EsqlIllegalArgumentException("Did not expect a non-aliased aggregation {}", agg); } @@ -61,9 +66,15 @@ private List aggsFromEmpty(List aggs) { /** * The folded aggregation output - this variant is for the coordinator/final. */ - protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { + protected void aggOutput( + FoldContext foldCtx, + NamedExpression agg, + AggregateFunction aggFunc, + BlockFactory blockFactory, + List blocks + ) { // look for count(literal) with literal != null - Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L : null; + Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L : null; var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(aggFunc.dataType()), 1); wrapper.accept(value); blocks.add(wrapper.builder().build()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java index 0bd98db1e1d7a..5a1677f2759e3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEquals.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.Iterator; @@ -41,17 +42,18 @@ public PropagateEquals() { super(OptimizerRules.TransformDirection.DOWN); } - public Expression rule(BinaryLogic e) { + @Override + public Expression rule(BinaryLogic e, LogicalOptimizerContext ctx) { if (e instanceof And) { - return propagate((And) e); + return propagate((And) e, ctx); } else if (e instanceof Or) { - return propagate((Or) e); + return propagate((Or) e, ctx); } return e; } // combine conjunction - private static Expression propagate(And and) { + private static Expression propagate(And and, LogicalOptimizerContext ctx) { List ranges = new ArrayList<>(); // Only equalities, not-equalities and inequalities with a foldable .right are extracted separately; // the others go into the general 'exps'. @@ -72,7 +74,7 @@ private static Expression propagate(And and) { if (otherEq.right().foldable() && DataType.isDateTime(otherEq.left().dataType()) == false) { for (BinaryComparison eq : equals) { if (otherEq.left().semanticEquals(eq.left())) { - Integer comp = BinaryComparison.compare(eq.right().fold(), otherEq.right().fold()); + Integer comp = BinaryComparison.compare(eq.right().fold(ctx.foldCtx()), otherEq.right().fold(ctx.foldCtx())); if (comp != null) { // var cannot be equal to two different values at the same time if (comp != 0) { @@ -108,7 +110,7 @@ private static Expression propagate(And and) { // check for (BinaryComparison eq : equals) { - Object eqValue = eq.right().fold(); + Object eqValue = eq.right().fold(ctx.foldCtx()); for (Iterator iterator = ranges.iterator(); iterator.hasNext();) { Range range = iterator.next(); @@ -116,7 +118,7 @@ private static Expression propagate(And and) { if (range.value().semanticEquals(eq.left())) { // if equals is outside the interval, evaluate the whole expression to FALSE if (range.lower().foldable()) { - Integer compare = BinaryComparison.compare(range.lower().fold(), eqValue); + Integer compare = BinaryComparison.compare(range.lower().fold(ctx.foldCtx()), eqValue); if (compare != null && ( // eq outside the lower boundary compare > 0 || @@ -126,7 +128,7 @@ private static Expression propagate(And and) { } } if (range.upper().foldable()) { - Integer compare = BinaryComparison.compare(range.upper().fold(), eqValue); + Integer compare = BinaryComparison.compare(range.upper().fold(ctx.foldCtx()), eqValue); if (compare != null && ( // eq outside the upper boundary compare < 0 || @@ -146,7 +148,7 @@ private static Expression propagate(And and) { for (Iterator iter = notEquals.iterator(); iter.hasNext();) { NotEquals neq = iter.next(); if (eq.left().semanticEquals(neq.left())) { - Integer comp = BinaryComparison.compare(eqValue, neq.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx())); if (comp != null) { if (comp == 0) { // clashing and conflicting: a = 1 AND a != 1 return new Literal(and.source(), Boolean.FALSE, DataType.BOOLEAN); @@ -162,7 +164,7 @@ private static Expression propagate(And and) { for (Iterator iter = inequalities.iterator(); iter.hasNext();) { BinaryComparison bc = iter.next(); if (eq.left().semanticEquals(bc.left())) { - Integer compare = BinaryComparison.compare(eqValue, bc.right().fold()); + Integer compare = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx())); if (compare != null) { if (bc instanceof LessThan || bc instanceof LessThanOrEqual) { // a = 2 AND a a < 3; a = 2 OR a < 1 -> nop // a = 2 OR 3 < a < 5 -> nop; a = 2 OR 1 < a < 3 -> 1 < a < 3; a = 2 OR 0 < a < 1 -> nop // a = 2 OR a != 2 -> TRUE; a = 2 OR a = 5 -> nop; a = 2 OR a != 5 -> a != 5 - private static Expression propagate(Or or) { + private static Expression propagate(Or or, LogicalOptimizerContext ctx) { List exps = new ArrayList<>(); List equals = new ArrayList<>(); // foldable right term Equals List notEquals = new ArrayList<>(); // foldable right term NotEquals @@ -230,13 +232,13 @@ private static Expression propagate(Or or) { // evaluate the impact of each Equal over the different types of Expressions for (Iterator iterEq = equals.iterator(); iterEq.hasNext();) { Equals eq = iterEq.next(); - Object eqValue = eq.right().fold(); + Object eqValue = eq.right().fold(ctx.foldCtx()); boolean removeEquals = false; // Equals OR NotEquals for (NotEquals neq : notEquals) { if (eq.left().semanticEquals(neq.left())) { // a = 2 OR a != ? -> ... - Integer comp = BinaryComparison.compare(eqValue, neq.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, neq.right().fold(ctx.foldCtx())); if (comp != null) { if (comp == 0) { // a = 2 OR a != 2 -> TRUE return TRUE; @@ -257,8 +259,12 @@ private static Expression propagate(Or or) { for (int i = 0; i < ranges.size(); i++) { // might modify list, so use index loop Range range = ranges.get(i); if (eq.left().semanticEquals(range.value())) { - Integer lowerComp = range.lower().foldable() ? BinaryComparison.compare(eqValue, range.lower().fold()) : null; - Integer upperComp = range.upper().foldable() ? BinaryComparison.compare(eqValue, range.upper().fold()) : null; + Integer lowerComp = range.lower().foldable() + ? BinaryComparison.compare(eqValue, range.lower().fold(ctx.foldCtx())) + : null; + Integer upperComp = range.upper().foldable() + ? BinaryComparison.compare(eqValue, range.upper().fold(ctx.foldCtx())) + : null; if (lowerComp != null && lowerComp == 0) { if (range.includeLower() == false) { // a = 2 OR 2 < a < ? -> 2 <= a < ? @@ -312,7 +318,7 @@ private static Expression propagate(Or or) { for (int i = 0; i < inequalities.size(); i++) { BinaryComparison bc = inequalities.get(i); if (eq.left().semanticEquals(bc.left())) { - Integer comp = BinaryComparison.compare(eqValue, bc.right().fold()); + Integer comp = BinaryComparison.compare(eqValue, bc.right().fold(ctx.foldCtx())); if (comp != null) { if (bc instanceof GreaterThan || bc instanceof GreaterThanOrEqual) { if (comp < 0) { // a = 1 OR a > 2 -> nop diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java index 73eaa9220fd84..66cdc992a91cb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEvalFoldables.java @@ -12,19 +12,20 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Eval; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.rule.Rule; +import org.elasticsearch.xpack.esql.rule.ParameterizedRule; /** * Replace any reference attribute with its source, if it does not affect the result. * This avoids ulterior look-ups between attributes and its source across nodes. */ -public final class PropagateEvalFoldables extends Rule { +public final class PropagateEvalFoldables extends ParameterizedRule { @Override - public LogicalPlan apply(LogicalPlan plan) { + public LogicalPlan apply(LogicalPlan plan, LogicalOptimizerContext ctx) { var collectRefs = new AttributeMap(); java.util.function.Function replaceReference = r -> collectRefs.resolve(r, r); @@ -39,7 +40,7 @@ public LogicalPlan apply(LogicalPlan plan) { shouldCollect = c.foldable(); } if (shouldCollect) { - collectRefs.put(a.toAttribute(), Literal.of(c)); + collectRefs.put(a.toAttribute(), Literal.of(ctx.foldCtx(), c)); } }); if (collectRefs.isEmpty()) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java index 738ca83b47e42..e3165180e331c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullable.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.LinkedHashSet; @@ -33,7 +34,7 @@ public PropagateNullable() { } @Override - public Expression rule(And and) { + public Expression rule(And and, LogicalOptimizerContext ctx) { List splits = Predicates.splitAnd(and); Set nullExpressions = new LinkedHashSet<>(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java index 1cacebdf27cd2..969a6bb713eca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimits.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.Eval; @@ -20,14 +21,18 @@ import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; -public final class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule { +public final class PushDownAndCombineLimits extends OptimizerRules.ParameterizedOptimizerRule { + + public PushDownAndCombineLimits() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - public LogicalPlan rule(Limit limit) { + public LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) { if (limit.child() instanceof Limit childLimit) { var limitSource = limit.limit(); - var l1 = (int) limitSource.fold(); - var l2 = (int) childLimit.limit().fold(); + var l1 = (int) limitSource.fold(ctx.foldCtx()); + var l2 = (int) childLimit.limit().fold(ctx.foldCtx()); return new Limit(limit.source(), Literal.of(limitSource, Math.min(l1, l2)), childLimit.child()); } else if (limit.child() instanceof UnaryPlan unary) { if (unary instanceof Eval || unary instanceof Project || unary instanceof RegexExtract || unary instanceof Enrich) { @@ -41,7 +46,7 @@ public LogicalPlan rule(Limit limit) { // we add an inner limit to MvExpand and just push down the existing limit, ie. // | MV_EXPAND | LIMIT N -> | LIMIT N | MV_EXPAND (with limit N) var limitSource = limit.limit(); - var limitVal = (int) limitSource.fold(); + var limitVal = (int) limitSource.fold(ctx.foldCtx()); Integer mvxLimit = mvx.limit(); if (mvxLimit == null || mvxLimit > limitVal) { mvx = new MvExpand(mvx.source(), mvx.child(), mvx.target(), mvx.expanded(), limitVal); @@ -54,8 +59,8 @@ public LogicalPlan rule(Limit limit) { else { Limit descendantLimit = descendantLimit(unary); if (descendantLimit != null) { - var l1 = (int) limit.limit().fold(); - var l2 = (int) descendantLimit.limit().fold(); + var l1 = (int) limit.limit().fold(ctx.foldCtx()); + var l2 = (int) descendantLimit.limit().fold(ctx.foldCtx()); if (l2 <= l1) { return new Limit(limit.source(), Literal.of(limit.limit(), l2), limit.child()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java index 1a8f8a164cc1b..7953b2b28eaaa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatch.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.parser.ParsingException; public final class ReplaceRegexMatch extends OptimizerRules.OptimizerExpressionRule> { @@ -23,7 +24,7 @@ public ReplaceRegexMatch() { } @Override - public Expression rule(RegexMatch regexMatch) { + public Expression rule(RegexMatch regexMatch, LogicalOptimizerContext ctx) { Expression e = regexMatch; StringPattern pattern = regexMatch.pattern(); boolean matchesAll; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java index eebeb1dc14f48..9e7b6ce80422d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRowAsLocalRelation.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; @@ -17,13 +18,16 @@ import java.util.ArrayList; import java.util.List; -public final class ReplaceRowAsLocalRelation extends OptimizerRules.OptimizerRule { +public final class ReplaceRowAsLocalRelation extends OptimizerRules.ParameterizedOptimizerRule { + public ReplaceRowAsLocalRelation() { + super(OptimizerRules.TransformDirection.DOWN); + } @Override - protected LogicalPlan rule(Row row) { + protected LogicalPlan rule(Row row, LogicalOptimizerContext context) { var fields = row.fields(); List values = new ArrayList<>(fields.size()); - fields.forEach(f -> values.add(f.child().fold())); + fields.forEach(f -> values.add(f.child().fold(context.foldCtx()))); var blocks = BlockUtils.fromListRow(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, values); return new LocalRelation(row.source(), row.output(), LocalSupplier.of(blocks)); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java index 2cafcc2e07052..a7e56a5f25fc8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStatsFilteredAggWithEval.java @@ -49,7 +49,7 @@ protected LogicalPlan rule(Aggregate aggregate) { && alias.child() instanceof AggregateFunction aggFunction && aggFunction.hasFilter() && aggFunction.filter() instanceof Literal literal - && Boolean.FALSE.equals(literal.fold())) { + && Boolean.FALSE.equals(literal.value())) { Object value = aggFunction instanceof Count || aggFunction instanceof CountDistinct ? 0L : null; Alias newAlias = alias.replaceChild(Literal.of(aggFunction, value)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java index 0fea7cf8ddc1f..053441bce5e1f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceStringCasingWithInsensitiveEquals.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; public class ReplaceStringCasingWithInsensitiveEquals extends OptimizerRules.OptimizerExpressionRule { @@ -26,30 +27,35 @@ public ReplaceStringCasingWithInsensitiveEquals() { } @Override - protected Expression rule(ScalarFunction sf) { + protected Expression rule(ScalarFunction sf, LogicalOptimizerContext ctx) { Expression e = sf; if (sf instanceof BinaryComparison bc) { - e = rewriteBinaryComparison(sf, bc, false); + e = rewriteBinaryComparison(ctx, sf, bc, false); } else if (sf instanceof Not not && not.field() instanceof BinaryComparison bc) { - e = rewriteBinaryComparison(sf, bc, true); + e = rewriteBinaryComparison(ctx, sf, bc, true); } return e; } - private static Expression rewriteBinaryComparison(ScalarFunction sf, BinaryComparison bc, boolean negated) { + private static Expression rewriteBinaryComparison( + LogicalOptimizerContext ctx, + ScalarFunction sf, + BinaryComparison bc, + boolean negated + ) { Expression e = sf; if (bc.left() instanceof ChangeCase changeCase && bc.right().foldable()) { if (bc instanceof Equals) { - e = replaceChangeCase(bc, changeCase, negated); + e = replaceChangeCase(ctx, bc, changeCase, negated); } else if (bc instanceof NotEquals) { // not actually used currently, `!=` is built as `NOT(==)` already - e = replaceChangeCase(bc, changeCase, negated == false); + e = replaceChangeCase(ctx, bc, changeCase, negated == false); } } return e; } - private static Expression replaceChangeCase(BinaryComparison bc, ChangeCase changeCase, boolean negated) { - var foldedRight = BytesRefs.toString(bc.right().fold()); + private static Expression replaceChangeCase(LogicalOptimizerContext ctx, BinaryComparison bc, ChangeCase changeCase, boolean negated) { + var foldedRight = BytesRefs.toString(bc.right().fold(ctx.foldCtx())); var field = unwrapCase(changeCase.field()); var e = changeCase.caseType().matchesCase(foldedRight) ? new InsensitiveEquals(bc.source(), field, bc.right()) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java index d3a9970896c16..60ff161651f2d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SimplifyComparisonsArithmetics.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -15,6 +16,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.BinaryComparisonInversible; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.time.DateTimeException; import java.util.List; @@ -41,20 +43,20 @@ public SimplifyComparisonsArithmetics(BiFunction ty } @Override - protected Expression rule(BinaryComparison bc) { + protected Expression rule(BinaryComparison bc, LogicalOptimizerContext ctx) { // optimize only once the expression has a literal on the right side of the binary comparison if (bc.right() instanceof Literal) { if (bc.left() instanceof ArithmeticOperation) { - return simplifyBinaryComparison(bc); + return simplifyBinaryComparison(ctx.foldCtx(), bc); } if (bc.left() instanceof Neg) { - return foldNegation(bc); + return foldNegation(ctx.foldCtx(), bc); } } return bc; } - private Expression simplifyBinaryComparison(BinaryComparison comparison) { + private Expression simplifyBinaryComparison(FoldContext foldContext, BinaryComparison comparison) { ArithmeticOperation operation = (ArithmeticOperation) comparison.left(); // Use symbol comp: SQL operations aren't available in this package (as dependencies) String opSymbol = operation.symbol(); @@ -64,9 +66,9 @@ private Expression simplifyBinaryComparison(BinaryComparison comparison) { } OperationSimplifier simplification = null; if (isMulOrDiv(opSymbol)) { - simplification = new MulDivSimplifier(comparison); + simplification = new MulDivSimplifier(foldContext, comparison); } else if (opSymbol.equals(ADD.symbol()) || opSymbol.equals(SUB.symbol())) { - simplification = new AddSubSimplifier(comparison); + simplification = new AddSubSimplifier(foldContext, comparison); } return (simplification == null || simplification.isUnsafe(typesCompatible)) ? comparison : simplification.apply(); @@ -76,16 +78,16 @@ private static boolean isMulOrDiv(String opSymbol) { return opSymbol.equals(MUL.symbol()) || opSymbol.equals(DIV.symbol()); } - private static Expression foldNegation(BinaryComparison bc) { + private static Expression foldNegation(FoldContext ctx, BinaryComparison bc) { Literal bcLiteral = (Literal) bc.right(); - Expression literalNeg = tryFolding(new Neg(bcLiteral.source(), bcLiteral)); + Expression literalNeg = tryFolding(ctx, new Neg(bcLiteral.source(), bcLiteral)); return literalNeg == null ? bc : bc.reverse().replaceChildren(asList(((Neg) bc.left()).field(), literalNeg)); } - private static Expression tryFolding(Expression expression) { + private static Expression tryFolding(FoldContext ctx, Expression expression) { if (expression.foldable()) { try { - expression = new Literal(expression.source(), expression.fold(), expression.dataType()); + expression = new Literal(expression.source(), expression.fold(ctx), expression.dataType()); } catch (ArithmeticException | DateTimeException e) { // null signals that folding would result in an over-/underflow (such as Long.MAX_VALUE+1); the optimisation is skipped. expression = null; @@ -95,6 +97,7 @@ private static Expression tryFolding(Expression expression) { } private abstract static class OperationSimplifier { + final FoldContext foldContext; final BinaryComparison comparison; final Literal bcLiteral; final ArithmeticOperation operation; @@ -102,7 +105,8 @@ private abstract static class OperationSimplifier { final Expression opRight; final Literal opLiteral; - OperationSimplifier(BinaryComparison comparison) { + OperationSimplifier(FoldContext foldContext, BinaryComparison comparison) { + this.foldContext = foldContext; this.comparison = comparison; operation = (ArithmeticOperation) comparison.left(); bcLiteral = (Literal) comparison.right(); @@ -151,7 +155,7 @@ final Expression apply() { Expression bcRightExpression = ((BinaryComparisonInversible) operation).binaryComparisonInverse() .create(bcl.source(), bcl, opRight); - bcRightExpression = tryFolding(bcRightExpression); + bcRightExpression = tryFolding(foldContext, bcRightExpression); return bcRightExpression != null ? postProcess((BinaryComparison) comparison.replaceChildren(List.of(opLeft, bcRightExpression))) : comparison; @@ -169,8 +173,8 @@ Expression postProcess(BinaryComparison binaryComparison) { private static class AddSubSimplifier extends OperationSimplifier { - AddSubSimplifier(BinaryComparison comparison) { - super(comparison); + AddSubSimplifier(FoldContext foldContext, BinaryComparison comparison) { + super(foldContext, comparison); } @Override @@ -182,7 +186,7 @@ boolean isOpUnsafe() { if (operation.symbol().equals(SUB.symbol()) && opRight instanceof Literal == false) { // such as: 1 - x > -MAX // if next simplification step would fail on overflow anyways, skip the optimisation already - return tryFolding(new Sub(EMPTY, opLeft, bcLiteral)) == null; + return tryFolding(foldContext, new Sub(EMPTY, opLeft, bcLiteral)) == null; } return false; @@ -194,8 +198,8 @@ private static class MulDivSimplifier extends OperationSimplifier { private final boolean isDiv; // and not MUL. private final int opRightSign; // sign of the right operand in: (left) (op) (right) (comp) (literal) - MulDivSimplifier(BinaryComparison comparison) { - super(comparison); + MulDivSimplifier(FoldContext foldContext, BinaryComparison comparison) { + super(foldContext, comparison); isDiv = operation.symbol().equals(DIV.symbol()); opRightSign = sign(opRight); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java index 5d98d941bb207..c6d62dee0ba42 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SkipQueryOnLimitZero.java @@ -7,14 +7,19 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; -public final class SkipQueryOnLimitZero extends OptimizerRules.OptimizerRule { +public final class SkipQueryOnLimitZero extends OptimizerRules.ParameterizedOptimizerRule { + public SkipQueryOnLimitZero() { + super(OptimizerRules.TransformDirection.DOWN); + } + @Override - protected LogicalPlan rule(Limit limit) { + protected LogicalPlan rule(Limit limit, LogicalOptimizerContext ctx) { if (limit.limit().foldable()) { - if (Integer.valueOf(0).equals((limit.limit().fold()))) { + if (Integer.valueOf(0).equals((limit.limit().fold(ctx.foldCtx())))) { return PruneEmptyPlans.skipPlan(limit); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java index 9e9ae6a9a559d..870464feb4867 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SplitInWithFoldableValue.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import java.util.ArrayList; import java.util.List; @@ -25,7 +26,7 @@ public SplitInWithFoldableValue() { } @Override - public Expression rule(In in) { + public Expression rule(In in, LogicalOptimizerContext ctx) { if (in.value().foldable()) { List foldables = new ArrayList<>(in.list().size()); List nonFoldables = new ArrayList<>(in.list().size()); @@ -36,7 +37,7 @@ public Expression rule(In in) { nonFoldables.add(e); } }); - if (foldables.size() > 0 && nonFoldables.size() > 0) { + if (foldables.isEmpty() == false && nonFoldables.isEmpty() == false) { In withFoldables = new In(in.source(), in.value(), foldables); In withoutFoldables = new In(in.source(), in.value(), nonFoldables); return new Or(in.source(), withFoldables, withoutFoldables); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java index c8369d2b08a34..62a00b79d7333 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteFilteredExpression.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.OptimizerExpressionRule; import org.elasticsearch.xpack.esql.optimizer.rules.logical.OptimizerRules.TransformDirection; @@ -21,7 +22,7 @@ public SubstituteFilteredExpression() { } @Override - protected Expression rule(FilteredExpression filteredExpression) { + protected Expression rule(FilteredExpression filteredExpression, LogicalOptimizerContext ctx) { return filteredExpression.surrogate(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java index 93512d80e1708..4b68ee941bc92 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.esql.optimizer.rules.logical; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; /** * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions, @@ -23,7 +24,7 @@ public SubstituteSpatialSurrogates() { } @Override - protected SpatialRelatesFunction rule(SpatialRelatesFunction function) { + protected SpatialRelatesFunction rule(SpatialRelatesFunction function, LogicalOptimizerContext ctx) { return function.surrogate(); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java index d29da1354ef3c..9259a50d5ff9e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/LocalPropagateEmptyRelation.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; @@ -25,19 +26,24 @@ * Local aggregation can only produce intermediate state that get wired into the global agg. */ public class LocalPropagateEmptyRelation extends PropagateEmptyRelation { - /** * Local variant of the aggregation that returns the intermediate value. */ @Override - protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { + protected void aggOutput( + FoldContext foldCtx, + NamedExpression agg, + AggregateFunction aggFunc, + BlockFactory blockFactory, + List blocks + ) { List output = AbstractPhysicalOperationProviders.intermediateAttributes(List.of(agg), List.of()); for (Attribute o : output) { DataType dataType = o.dataType(); // boolean right now is used for the internal #seen so always return true var value = dataType == DataType.BOOLEAN ? true // look for count(literal) with literal != null - : aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L + : aggFunc instanceof Count count && (count.foldable() == false || count.fold(foldCtx) != null) ? 0L // otherwise nullify : null; var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(dataType), 1); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java index dfb1dbc8bc8f3..afeab28745c65 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/EnableSpatialDistancePushdown.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -76,22 +77,33 @@ public class EnableSpatialDistancePushdown extends PhysicalOptimizerRules.Parame protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext ctx) { PhysicalPlan plan = filterExec; if (filterExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewrite(filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewrite(ctx.foldCtx(), filterExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); } else if (filterExec.child() instanceof EvalExec evalExec && evalExec.child() instanceof EsQueryExec esQueryExec) { - plan = rewriteBySplittingFilter(filterExec, evalExec, esQueryExec, LucenePushdownPredicates.from(ctx.searchStats())); + plan = rewriteBySplittingFilter( + ctx.foldCtx(), + filterExec, + evalExec, + esQueryExec, + LucenePushdownPredicates.from(ctx.searchStats()) + ); } return plan; } - private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, LucenePushdownPredicates lucenePushdownPredicates) { + private FilterExec rewrite( + FoldContext ctx, + FilterExec filterExec, + EsQueryExec esQueryExec, + LucenePushdownPredicates lucenePushdownPredicates + ) { // Find and rewrite any binary comparisons that involve a distance function and a literal var rewritten = filterExec.condition().transformDown(EsqlBinaryComparison.class, comparison -> { ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); if (comparison.left() instanceof StDistance dist && comparison.right().foldable()) { - return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType); } else if (comparison.right() instanceof StDistance dist && comparison.left().foldable()) { - return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); } return comparison; }); @@ -120,6 +132,7 @@ private FilterExec rewrite(FilterExec filterExec, EsQueryExec esQueryExec, Lucen * */ private PhysicalPlan rewriteBySplittingFilter( + FoldContext ctx, FilterExec filterExec, EvalExec evalExec, EsQueryExec esQueryExec, @@ -142,7 +155,7 @@ private PhysicalPlan rewriteBySplittingFilter( for (Expression exp : splitAnd(filterExec.condition())) { Expression resExp = exp.transformUp(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); // Find and rewrite any binary comparisons that involve a distance function and a literal - var rewritten = rewriteDistanceFilters(resExp, distances); + var rewritten = rewriteDistanceFilters(ctx, resExp, distances); // If all pushable StDistance functions were found and re-written, we need to re-write the FILTER/EVAL combination if (rewritten.equals(resExp) == false && canPushToSource(rewritten, lucenePushdownPredicates)) { pushable.add(rewritten); @@ -181,40 +194,42 @@ private Map getPushableDistances(List aliases, Lucene return distances; } - private Expression rewriteDistanceFilters(Expression expr, Map distances) { + private Expression rewriteDistanceFilters(FoldContext ctx, Expression expr, Map distances) { return expr.transformDown(EsqlBinaryComparison.class, comparison -> { ComparisonType comparisonType = ComparisonType.from(comparison.getFunctionType()); if (comparison.left() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.right().foldable()) { StDistance dist = distances.get(r.id()); - return rewriteComparison(comparison, dist, comparison.right(), comparisonType); + return rewriteComparison(ctx, comparison, dist, comparison.right(), comparisonType); } else if (comparison.right() instanceof ReferenceAttribute r && distances.containsKey(r.id()) && comparison.left().foldable()) { StDistance dist = distances.get(r.id()); - return rewriteComparison(comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); + return rewriteComparison(ctx, comparison, dist, comparison.left(), ComparisonType.invert(comparisonType)); } return comparison; }); } private Expression rewriteComparison( + FoldContext ctx, EsqlBinaryComparison comparison, StDistance dist, Expression literal, ComparisonType comparisonType ) { - Object value = literal.fold(); + Object value = literal.fold(ctx); if (value instanceof Number number) { if (dist.right().foldable()) { - return rewriteDistanceFilter(comparison, dist.left(), dist.right(), number, comparisonType); + return rewriteDistanceFilter(ctx, comparison, dist.left(), dist.right(), number, comparisonType); } else if (dist.left().foldable()) { - return rewriteDistanceFilter(comparison, dist.right(), dist.left(), number, comparisonType); + return rewriteDistanceFilter(ctx, comparison, dist.right(), dist.left(), number, comparisonType); } } return comparison; } private Expression rewriteDistanceFilter( + FoldContext ctx, EsqlBinaryComparison comparison, Expression spatialExp, Expression literalExp, @@ -222,7 +237,7 @@ private Expression rewriteDistanceFilter( ComparisonType comparisonType ) { DataType shapeDataType = getShapeDataType(spatialExp); - Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(literalExp); + Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, literalExp); if (geometry instanceof Point point) { double distance = number.doubleValue(); Source source = comparison.source(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java index 2b531257e594a..24df3c1db234e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSource.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeMap; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -61,7 +62,7 @@ public class PushTopNToSource extends PhysicalOptimizerRules.ParameterizedOptimi @Override protected PhysicalPlan rule(TopNExec topNExec, LocalPhysicalOptimizerContext ctx) { - Pushable pushable = evaluatePushable(topNExec, LucenePushdownPredicates.from(ctx.searchStats())); + Pushable pushable = evaluatePushable(ctx.foldCtx(), topNExec, LucenePushdownPredicates.from(ctx.searchStats())); return pushable.rewrite(topNExec); } @@ -95,18 +96,18 @@ private EsQueryExec.Sort sort() { return new EsQueryExec.GeoDistanceSort(fieldAttribute.exactAttribute(), order.direction(), point.getLat(), point.getLon()); } - private static PushableGeoDistance from(StDistance distance, Order order) { + private static PushableGeoDistance from(FoldContext ctx, StDistance distance, Order order) { if (distance.left() instanceof Attribute attr && distance.right().foldable()) { - return from(attr, distance.right(), order); + return from(ctx, attr, distance.right(), order); } else if (distance.right() instanceof Attribute attr && distance.left().foldable()) { - return from(attr, distance.left(), order); + return from(ctx, attr, distance.left(), order); } return null; } - private static PushableGeoDistance from(Attribute attr, Expression foldable, Order order) { + private static PushableGeoDistance from(FoldContext ctx, Attribute attr, Expression foldable, Order order) { if (attr instanceof FieldAttribute fieldAttribute) { - Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(foldable); + Geometry geometry = SpatialRelatesUtils.makeGeometryFromLiteral(ctx, foldable); if (geometry instanceof Point point) { return new PushableGeoDistance(fieldAttribute, order, point); } @@ -122,7 +123,7 @@ public PhysicalPlan rewrite(TopNExec topNExec) { } } - private static Pushable evaluatePushable(TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) { + private static Pushable evaluatePushable(FoldContext ctx, TopNExec topNExec, LucenePushdownPredicates lucenePushdownPredicates) { PhysicalPlan child = topNExec.child(); if (child instanceof EsQueryExec queryExec && queryExec.canPushSorts() @@ -164,7 +165,7 @@ && canPushDownOrders(topNExec.order(), lucenePushdownPredicates)) { if (distances.containsKey(resolvedAttribute.id())) { StDistance distance = distances.get(resolvedAttribute.id()); StDistance d = (StDistance) distance.transformDown(ReferenceAttribute.class, r -> aliasReplacedBy.resolve(r, r)); - PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(d, order); + PushableGeoDistance pushableGeoDistance = PushableGeoDistance.from(ctx, d, order); if (pushableGeoDistance != null) { pushableSorts.add(pushableGeoDistance.sort()); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp index c5b37fa411f65..492df7fbc1608 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.interp @@ -283,8 +283,6 @@ indexPattern clusterString indexString metadata -metadataOption -deprecated_metadata metricsCommand evalCommand statsCommand @@ -330,4 +328,4 @@ joinPredicate atn: -[4, 1, 128, 639, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 142, 8, 1, 10, 1, 12, 1, 145, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 153, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 173, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 185, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 192, 8, 5, 10, 5, 12, 5, 195, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 202, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 207, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 215, 8, 5, 10, 5, 12, 5, 218, 9, 5, 1, 6, 1, 6, 3, 6, 222, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 229, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 234, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 239, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 249, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 255, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 263, 8, 9, 10, 9, 12, 9, 266, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 276, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 281, 8, 10, 10, 10, 12, 10, 284, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 292, 8, 11, 10, 11, 12, 11, 295, 9, 11, 3, 11, 297, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 311, 8, 15, 10, 15, 12, 15, 314, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 319, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 327, 8, 17, 10, 17, 12, 17, 330, 9, 17, 1, 17, 3, 17, 333, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 338, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 3, 21, 348, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 354, 8, 22, 10, 22, 12, 22, 357, 9, 22, 1, 23, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 1, 24, 1, 24, 5, 24, 367, 8, 24, 10, 24, 12, 24, 370, 9, 24, 1, 24, 3, 24, 373, 8, 24, 1, 24, 1, 24, 3, 24, 377, 8, 24, 1, 25, 1, 25, 1, 25, 1, 26, 1, 26, 3, 26, 384, 8, 26, 1, 26, 1, 26, 3, 26, 388, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 393, 8, 27, 10, 27, 12, 27, 396, 9, 27, 1, 28, 1, 28, 1, 28, 3, 28, 401, 8, 28, 1, 29, 1, 29, 1, 29, 5, 29, 406, 8, 29, 10, 29, 12, 29, 409, 9, 29, 1, 30, 1, 30, 1, 30, 5, 30, 414, 8, 30, 10, 30, 12, 30, 417, 9, 30, 1, 31, 1, 31, 1, 31, 5, 31, 422, 8, 31, 10, 31, 12, 31, 425, 9, 31, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 3, 33, 432, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 447, 8, 34, 10, 34, 12, 34, 450, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 458, 8, 34, 10, 34, 12, 34, 461, 9, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 5, 34, 469, 8, 34, 10, 34, 12, 34, 472, 9, 34, 1, 34, 1, 34, 3, 34, 476, 8, 34, 1, 35, 1, 35, 3, 35, 480, 8, 35, 1, 36, 1, 36, 1, 36, 3, 36, 485, 8, 36, 1, 37, 1, 37, 1, 37, 1, 38, 1, 38, 1, 38, 1, 38, 5, 38, 494, 8, 38, 10, 38, 12, 38, 497, 9, 38, 1, 39, 1, 39, 3, 39, 501, 8, 39, 1, 39, 1, 39, 3, 39, 505, 8, 39, 1, 40, 1, 40, 1, 40, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 517, 8, 42, 10, 42, 12, 42, 520, 9, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 530, 8, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 542, 8, 47, 10, 47, 12, 47, 545, 9, 47, 1, 48, 1, 48, 1, 48, 1, 48, 1, 49, 1, 49, 1, 50, 1, 50, 3, 50, 555, 8, 50, 1, 51, 3, 51, 558, 8, 51, 1, 51, 1, 51, 1, 52, 3, 52, 563, 8, 52, 1, 52, 1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 585, 8, 58, 1, 58, 1, 58, 1, 58, 1, 58, 5, 58, 591, 8, 58, 10, 58, 12, 58, 594, 9, 58, 3, 58, 596, 8, 58, 1, 59, 1, 59, 1, 59, 3, 59, 601, 8, 59, 1, 59, 1, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 3, 62, 617, 8, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 63, 1, 63, 1, 63, 3, 63, 626, 8, 63, 1, 64, 1, 64, 1, 64, 1, 64, 5, 64, 632, 8, 64, 10, 64, 12, 64, 635, 9, 64, 1, 65, 1, 65, 1, 65, 0, 4, 2, 10, 18, 20, 66, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 128, 130, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 665, 0, 132, 1, 0, 0, 0, 2, 135, 1, 0, 0, 0, 4, 152, 1, 0, 0, 0, 6, 172, 1, 0, 0, 0, 8, 174, 1, 0, 0, 0, 10, 206, 1, 0, 0, 0, 12, 233, 1, 0, 0, 0, 14, 235, 1, 0, 0, 0, 16, 248, 1, 0, 0, 0, 18, 254, 1, 0, 0, 0, 20, 275, 1, 0, 0, 0, 22, 285, 1, 0, 0, 0, 24, 300, 1, 0, 0, 0, 26, 302, 1, 0, 0, 0, 28, 304, 1, 0, 0, 0, 30, 307, 1, 0, 0, 0, 32, 318, 1, 0, 0, 0, 34, 322, 1, 0, 0, 0, 36, 337, 1, 0, 0, 0, 38, 341, 1, 0, 0, 0, 40, 343, 1, 0, 0, 0, 42, 347, 1, 0, 0, 0, 44, 349, 1, 0, 0, 0, 46, 358, 1, 0, 0, 0, 48, 362, 1, 0, 0, 0, 50, 378, 1, 0, 0, 0, 52, 381, 1, 0, 0, 0, 54, 389, 1, 0, 0, 0, 56, 397, 1, 0, 0, 0, 58, 402, 1, 0, 0, 0, 60, 410, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 431, 1, 0, 0, 0, 68, 475, 1, 0, 0, 0, 70, 479, 1, 0, 0, 0, 72, 484, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 489, 1, 0, 0, 0, 78, 498, 1, 0, 0, 0, 80, 506, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 512, 1, 0, 0, 0, 86, 521, 1, 0, 0, 0, 88, 525, 1, 0, 0, 0, 90, 531, 1, 0, 0, 0, 92, 535, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 546, 1, 0, 0, 0, 98, 550, 1, 0, 0, 0, 100, 554, 1, 0, 0, 0, 102, 557, 1, 0, 0, 0, 104, 562, 1, 0, 0, 0, 106, 566, 1, 0, 0, 0, 108, 568, 1, 0, 0, 0, 110, 570, 1, 0, 0, 0, 112, 573, 1, 0, 0, 0, 114, 577, 1, 0, 0, 0, 116, 580, 1, 0, 0, 0, 118, 600, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 609, 1, 0, 0, 0, 124, 616, 1, 0, 0, 0, 126, 622, 1, 0, 0, 0, 128, 627, 1, 0, 0, 0, 130, 636, 1, 0, 0, 0, 132, 133, 3, 2, 1, 0, 133, 134, 5, 0, 0, 1, 134, 1, 1, 0, 0, 0, 135, 136, 6, 1, -1, 0, 136, 137, 3, 4, 2, 0, 137, 143, 1, 0, 0, 0, 138, 139, 10, 1, 0, 0, 139, 140, 5, 29, 0, 0, 140, 142, 3, 6, 3, 0, 141, 138, 1, 0, 0, 0, 142, 145, 1, 0, 0, 0, 143, 141, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 3, 1, 0, 0, 0, 145, 143, 1, 0, 0, 0, 146, 153, 3, 110, 55, 0, 147, 153, 3, 34, 17, 0, 148, 153, 3, 28, 14, 0, 149, 153, 3, 114, 57, 0, 150, 151, 4, 2, 1, 0, 151, 153, 3, 48, 24, 0, 152, 146, 1, 0, 0, 0, 152, 147, 1, 0, 0, 0, 152, 148, 1, 0, 0, 0, 152, 149, 1, 0, 0, 0, 152, 150, 1, 0, 0, 0, 153, 5, 1, 0, 0, 0, 154, 173, 3, 50, 25, 0, 155, 173, 3, 8, 4, 0, 156, 173, 3, 80, 40, 0, 157, 173, 3, 74, 37, 0, 158, 173, 3, 52, 26, 0, 159, 173, 3, 76, 38, 0, 160, 173, 3, 82, 41, 0, 161, 173, 3, 84, 42, 0, 162, 173, 3, 88, 44, 0, 163, 173, 3, 90, 45, 0, 164, 173, 3, 116, 58, 0, 165, 173, 3, 92, 46, 0, 166, 167, 4, 3, 2, 0, 167, 173, 3, 122, 61, 0, 168, 169, 4, 3, 3, 0, 169, 173, 3, 120, 60, 0, 170, 171, 4, 3, 4, 0, 171, 173, 3, 124, 62, 0, 172, 154, 1, 0, 0, 0, 172, 155, 1, 0, 0, 0, 172, 156, 1, 0, 0, 0, 172, 157, 1, 0, 0, 0, 172, 158, 1, 0, 0, 0, 172, 159, 1, 0, 0, 0, 172, 160, 1, 0, 0, 0, 172, 161, 1, 0, 0, 0, 172, 162, 1, 0, 0, 0, 172, 163, 1, 0, 0, 0, 172, 164, 1, 0, 0, 0, 172, 165, 1, 0, 0, 0, 172, 166, 1, 0, 0, 0, 172, 168, 1, 0, 0, 0, 172, 170, 1, 0, 0, 0, 173, 7, 1, 0, 0, 0, 174, 175, 5, 16, 0, 0, 175, 176, 3, 10, 5, 0, 176, 9, 1, 0, 0, 0, 177, 178, 6, 5, -1, 0, 178, 179, 5, 49, 0, 0, 179, 207, 3, 10, 5, 8, 180, 207, 3, 16, 8, 0, 181, 207, 3, 12, 6, 0, 182, 184, 3, 16, 8, 0, 183, 185, 5, 49, 0, 0, 184, 183, 1, 0, 0, 0, 184, 185, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 187, 5, 44, 0, 0, 187, 188, 5, 48, 0, 0, 188, 193, 3, 16, 8, 0, 189, 190, 5, 39, 0, 0, 190, 192, 3, 16, 8, 0, 191, 189, 1, 0, 0, 0, 192, 195, 1, 0, 0, 0, 193, 191, 1, 0, 0, 0, 193, 194, 1, 0, 0, 0, 194, 196, 1, 0, 0, 0, 195, 193, 1, 0, 0, 0, 196, 197, 5, 55, 0, 0, 197, 207, 1, 0, 0, 0, 198, 199, 3, 16, 8, 0, 199, 201, 5, 45, 0, 0, 200, 202, 5, 49, 0, 0, 201, 200, 1, 0, 0, 0, 201, 202, 1, 0, 0, 0, 202, 203, 1, 0, 0, 0, 203, 204, 5, 50, 0, 0, 204, 207, 1, 0, 0, 0, 205, 207, 3, 14, 7, 0, 206, 177, 1, 0, 0, 0, 206, 180, 1, 0, 0, 0, 206, 181, 1, 0, 0, 0, 206, 182, 1, 0, 0, 0, 206, 198, 1, 0, 0, 0, 206, 205, 1, 0, 0, 0, 207, 216, 1, 0, 0, 0, 208, 209, 10, 5, 0, 0, 209, 210, 5, 34, 0, 0, 210, 215, 3, 10, 5, 6, 211, 212, 10, 4, 0, 0, 212, 213, 5, 52, 0, 0, 213, 215, 3, 10, 5, 5, 214, 208, 1, 0, 0, 0, 214, 211, 1, 0, 0, 0, 215, 218, 1, 0, 0, 0, 216, 214, 1, 0, 0, 0, 216, 217, 1, 0, 0, 0, 217, 11, 1, 0, 0, 0, 218, 216, 1, 0, 0, 0, 219, 221, 3, 16, 8, 0, 220, 222, 5, 49, 0, 0, 221, 220, 1, 0, 0, 0, 221, 222, 1, 0, 0, 0, 222, 223, 1, 0, 0, 0, 223, 224, 5, 47, 0, 0, 224, 225, 3, 106, 53, 0, 225, 234, 1, 0, 0, 0, 226, 228, 3, 16, 8, 0, 227, 229, 5, 49, 0, 0, 228, 227, 1, 0, 0, 0, 228, 229, 1, 0, 0, 0, 229, 230, 1, 0, 0, 0, 230, 231, 5, 54, 0, 0, 231, 232, 3, 106, 53, 0, 232, 234, 1, 0, 0, 0, 233, 219, 1, 0, 0, 0, 233, 226, 1, 0, 0, 0, 234, 13, 1, 0, 0, 0, 235, 238, 3, 58, 29, 0, 236, 237, 5, 37, 0, 0, 237, 239, 3, 26, 13, 0, 238, 236, 1, 0, 0, 0, 238, 239, 1, 0, 0, 0, 239, 240, 1, 0, 0, 0, 240, 241, 5, 38, 0, 0, 241, 242, 3, 68, 34, 0, 242, 15, 1, 0, 0, 0, 243, 249, 3, 18, 9, 0, 244, 245, 3, 18, 9, 0, 245, 246, 3, 108, 54, 0, 246, 247, 3, 18, 9, 0, 247, 249, 1, 0, 0, 0, 248, 243, 1, 0, 0, 0, 248, 244, 1, 0, 0, 0, 249, 17, 1, 0, 0, 0, 250, 251, 6, 9, -1, 0, 251, 255, 3, 20, 10, 0, 252, 253, 7, 0, 0, 0, 253, 255, 3, 18, 9, 3, 254, 250, 1, 0, 0, 0, 254, 252, 1, 0, 0, 0, 255, 264, 1, 0, 0, 0, 256, 257, 10, 2, 0, 0, 257, 258, 7, 1, 0, 0, 258, 263, 3, 18, 9, 3, 259, 260, 10, 1, 0, 0, 260, 261, 7, 0, 0, 0, 261, 263, 3, 18, 9, 2, 262, 256, 1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 266, 1, 0, 0, 0, 264, 262, 1, 0, 0, 0, 264, 265, 1, 0, 0, 0, 265, 19, 1, 0, 0, 0, 266, 264, 1, 0, 0, 0, 267, 268, 6, 10, -1, 0, 268, 276, 3, 68, 34, 0, 269, 276, 3, 58, 29, 0, 270, 276, 3, 22, 11, 0, 271, 272, 5, 48, 0, 0, 272, 273, 3, 10, 5, 0, 273, 274, 5, 55, 0, 0, 274, 276, 1, 0, 0, 0, 275, 267, 1, 0, 0, 0, 275, 269, 1, 0, 0, 0, 275, 270, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 282, 1, 0, 0, 0, 277, 278, 10, 1, 0, 0, 278, 279, 5, 37, 0, 0, 279, 281, 3, 26, 13, 0, 280, 277, 1, 0, 0, 0, 281, 284, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 282, 283, 1, 0, 0, 0, 283, 21, 1, 0, 0, 0, 284, 282, 1, 0, 0, 0, 285, 286, 3, 24, 12, 0, 286, 296, 5, 48, 0, 0, 287, 297, 5, 66, 0, 0, 288, 293, 3, 10, 5, 0, 289, 290, 5, 39, 0, 0, 290, 292, 3, 10, 5, 0, 291, 289, 1, 0, 0, 0, 292, 295, 1, 0, 0, 0, 293, 291, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 297, 1, 0, 0, 0, 295, 293, 1, 0, 0, 0, 296, 287, 1, 0, 0, 0, 296, 288, 1, 0, 0, 0, 296, 297, 1, 0, 0, 0, 297, 298, 1, 0, 0, 0, 298, 299, 5, 55, 0, 0, 299, 23, 1, 0, 0, 0, 300, 301, 3, 72, 36, 0, 301, 25, 1, 0, 0, 0, 302, 303, 3, 64, 32, 0, 303, 27, 1, 0, 0, 0, 304, 305, 5, 12, 0, 0, 305, 306, 3, 30, 15, 0, 306, 29, 1, 0, 0, 0, 307, 312, 3, 32, 16, 0, 308, 309, 5, 39, 0, 0, 309, 311, 3, 32, 16, 0, 310, 308, 1, 0, 0, 0, 311, 314, 1, 0, 0, 0, 312, 310, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 31, 1, 0, 0, 0, 314, 312, 1, 0, 0, 0, 315, 316, 3, 58, 29, 0, 316, 317, 5, 36, 0, 0, 317, 319, 1, 0, 0, 0, 318, 315, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 320, 1, 0, 0, 0, 320, 321, 3, 10, 5, 0, 321, 33, 1, 0, 0, 0, 322, 323, 5, 6, 0, 0, 323, 328, 3, 36, 18, 0, 324, 325, 5, 39, 0, 0, 325, 327, 3, 36, 18, 0, 326, 324, 1, 0, 0, 0, 327, 330, 1, 0, 0, 0, 328, 326, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 332, 1, 0, 0, 0, 330, 328, 1, 0, 0, 0, 331, 333, 3, 42, 21, 0, 332, 331, 1, 0, 0, 0, 332, 333, 1, 0, 0, 0, 333, 35, 1, 0, 0, 0, 334, 335, 3, 38, 19, 0, 335, 336, 5, 38, 0, 0, 336, 338, 1, 0, 0, 0, 337, 334, 1, 0, 0, 0, 337, 338, 1, 0, 0, 0, 338, 339, 1, 0, 0, 0, 339, 340, 3, 40, 20, 0, 340, 37, 1, 0, 0, 0, 341, 342, 5, 81, 0, 0, 342, 39, 1, 0, 0, 0, 343, 344, 7, 2, 0, 0, 344, 41, 1, 0, 0, 0, 345, 348, 3, 44, 22, 0, 346, 348, 3, 46, 23, 0, 347, 345, 1, 0, 0, 0, 347, 346, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 350, 5, 80, 0, 0, 350, 355, 5, 81, 0, 0, 351, 352, 5, 39, 0, 0, 352, 354, 5, 81, 0, 0, 353, 351, 1, 0, 0, 0, 354, 357, 1, 0, 0, 0, 355, 353, 1, 0, 0, 0, 355, 356, 1, 0, 0, 0, 356, 45, 1, 0, 0, 0, 357, 355, 1, 0, 0, 0, 358, 359, 5, 70, 0, 0, 359, 360, 3, 44, 22, 0, 360, 361, 5, 71, 0, 0, 361, 47, 1, 0, 0, 0, 362, 363, 5, 19, 0, 0, 363, 368, 3, 36, 18, 0, 364, 365, 5, 39, 0, 0, 365, 367, 3, 36, 18, 0, 366, 364, 1, 0, 0, 0, 367, 370, 1, 0, 0, 0, 368, 366, 1, 0, 0, 0, 368, 369, 1, 0, 0, 0, 369, 372, 1, 0, 0, 0, 370, 368, 1, 0, 0, 0, 371, 373, 3, 54, 27, 0, 372, 371, 1, 0, 0, 0, 372, 373, 1, 0, 0, 0, 373, 376, 1, 0, 0, 0, 374, 375, 5, 33, 0, 0, 375, 377, 3, 30, 15, 0, 376, 374, 1, 0, 0, 0, 376, 377, 1, 0, 0, 0, 377, 49, 1, 0, 0, 0, 378, 379, 5, 4, 0, 0, 379, 380, 3, 30, 15, 0, 380, 51, 1, 0, 0, 0, 381, 383, 5, 15, 0, 0, 382, 384, 3, 54, 27, 0, 383, 382, 1, 0, 0, 0, 383, 384, 1, 0, 0, 0, 384, 387, 1, 0, 0, 0, 385, 386, 5, 33, 0, 0, 386, 388, 3, 30, 15, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 53, 1, 0, 0, 0, 389, 394, 3, 56, 28, 0, 390, 391, 5, 39, 0, 0, 391, 393, 3, 56, 28, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 55, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 400, 3, 32, 16, 0, 398, 399, 5, 16, 0, 0, 399, 401, 3, 10, 5, 0, 400, 398, 1, 0, 0, 0, 400, 401, 1, 0, 0, 0, 401, 57, 1, 0, 0, 0, 402, 407, 3, 72, 36, 0, 403, 404, 5, 41, 0, 0, 404, 406, 3, 72, 36, 0, 405, 403, 1, 0, 0, 0, 406, 409, 1, 0, 0, 0, 407, 405, 1, 0, 0, 0, 407, 408, 1, 0, 0, 0, 408, 59, 1, 0, 0, 0, 409, 407, 1, 0, 0, 0, 410, 415, 3, 66, 33, 0, 411, 412, 5, 41, 0, 0, 412, 414, 3, 66, 33, 0, 413, 411, 1, 0, 0, 0, 414, 417, 1, 0, 0, 0, 415, 413, 1, 0, 0, 0, 415, 416, 1, 0, 0, 0, 416, 61, 1, 0, 0, 0, 417, 415, 1, 0, 0, 0, 418, 423, 3, 60, 30, 0, 419, 420, 5, 39, 0, 0, 420, 422, 3, 60, 30, 0, 421, 419, 1, 0, 0, 0, 422, 425, 1, 0, 0, 0, 423, 421, 1, 0, 0, 0, 423, 424, 1, 0, 0, 0, 424, 63, 1, 0, 0, 0, 425, 423, 1, 0, 0, 0, 426, 427, 7, 3, 0, 0, 427, 65, 1, 0, 0, 0, 428, 432, 5, 85, 0, 0, 429, 430, 4, 33, 10, 0, 430, 432, 3, 70, 35, 0, 431, 428, 1, 0, 0, 0, 431, 429, 1, 0, 0, 0, 432, 67, 1, 0, 0, 0, 433, 476, 5, 50, 0, 0, 434, 435, 3, 104, 52, 0, 435, 436, 5, 72, 0, 0, 436, 476, 1, 0, 0, 0, 437, 476, 3, 102, 51, 0, 438, 476, 3, 104, 52, 0, 439, 476, 3, 98, 49, 0, 440, 476, 3, 70, 35, 0, 441, 476, 3, 106, 53, 0, 442, 443, 5, 70, 0, 0, 443, 448, 3, 100, 50, 0, 444, 445, 5, 39, 0, 0, 445, 447, 3, 100, 50, 0, 446, 444, 1, 0, 0, 0, 447, 450, 1, 0, 0, 0, 448, 446, 1, 0, 0, 0, 448, 449, 1, 0, 0, 0, 449, 451, 1, 0, 0, 0, 450, 448, 1, 0, 0, 0, 451, 452, 5, 71, 0, 0, 452, 476, 1, 0, 0, 0, 453, 454, 5, 70, 0, 0, 454, 459, 3, 98, 49, 0, 455, 456, 5, 39, 0, 0, 456, 458, 3, 98, 49, 0, 457, 455, 1, 0, 0, 0, 458, 461, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 459, 460, 1, 0, 0, 0, 460, 462, 1, 0, 0, 0, 461, 459, 1, 0, 0, 0, 462, 463, 5, 71, 0, 0, 463, 476, 1, 0, 0, 0, 464, 465, 5, 70, 0, 0, 465, 470, 3, 106, 53, 0, 466, 467, 5, 39, 0, 0, 467, 469, 3, 106, 53, 0, 468, 466, 1, 0, 0, 0, 469, 472, 1, 0, 0, 0, 470, 468, 1, 0, 0, 0, 470, 471, 1, 0, 0, 0, 471, 473, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 474, 5, 71, 0, 0, 474, 476, 1, 0, 0, 0, 475, 433, 1, 0, 0, 0, 475, 434, 1, 0, 0, 0, 475, 437, 1, 0, 0, 0, 475, 438, 1, 0, 0, 0, 475, 439, 1, 0, 0, 0, 475, 440, 1, 0, 0, 0, 475, 441, 1, 0, 0, 0, 475, 442, 1, 0, 0, 0, 475, 453, 1, 0, 0, 0, 475, 464, 1, 0, 0, 0, 476, 69, 1, 0, 0, 0, 477, 480, 5, 53, 0, 0, 478, 480, 5, 69, 0, 0, 479, 477, 1, 0, 0, 0, 479, 478, 1, 0, 0, 0, 480, 71, 1, 0, 0, 0, 481, 485, 3, 64, 32, 0, 482, 483, 4, 36, 11, 0, 483, 485, 3, 70, 35, 0, 484, 481, 1, 0, 0, 0, 484, 482, 1, 0, 0, 0, 485, 73, 1, 0, 0, 0, 486, 487, 5, 9, 0, 0, 487, 488, 5, 31, 0, 0, 488, 75, 1, 0, 0, 0, 489, 490, 5, 14, 0, 0, 490, 495, 3, 78, 39, 0, 491, 492, 5, 39, 0, 0, 492, 494, 3, 78, 39, 0, 493, 491, 1, 0, 0, 0, 494, 497, 1, 0, 0, 0, 495, 493, 1, 0, 0, 0, 495, 496, 1, 0, 0, 0, 496, 77, 1, 0, 0, 0, 497, 495, 1, 0, 0, 0, 498, 500, 3, 10, 5, 0, 499, 501, 7, 4, 0, 0, 500, 499, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 504, 1, 0, 0, 0, 502, 503, 5, 51, 0, 0, 503, 505, 7, 5, 0, 0, 504, 502, 1, 0, 0, 0, 504, 505, 1, 0, 0, 0, 505, 79, 1, 0, 0, 0, 506, 507, 5, 8, 0, 0, 507, 508, 3, 62, 31, 0, 508, 81, 1, 0, 0, 0, 509, 510, 5, 2, 0, 0, 510, 511, 3, 62, 31, 0, 511, 83, 1, 0, 0, 0, 512, 513, 5, 11, 0, 0, 513, 518, 3, 86, 43, 0, 514, 515, 5, 39, 0, 0, 515, 517, 3, 86, 43, 0, 516, 514, 1, 0, 0, 0, 517, 520, 1, 0, 0, 0, 518, 516, 1, 0, 0, 0, 518, 519, 1, 0, 0, 0, 519, 85, 1, 0, 0, 0, 520, 518, 1, 0, 0, 0, 521, 522, 3, 60, 30, 0, 522, 523, 5, 89, 0, 0, 523, 524, 3, 60, 30, 0, 524, 87, 1, 0, 0, 0, 525, 526, 5, 1, 0, 0, 526, 527, 3, 20, 10, 0, 527, 529, 3, 106, 53, 0, 528, 530, 3, 94, 47, 0, 529, 528, 1, 0, 0, 0, 529, 530, 1, 0, 0, 0, 530, 89, 1, 0, 0, 0, 531, 532, 5, 7, 0, 0, 532, 533, 3, 20, 10, 0, 533, 534, 3, 106, 53, 0, 534, 91, 1, 0, 0, 0, 535, 536, 5, 10, 0, 0, 536, 537, 3, 58, 29, 0, 537, 93, 1, 0, 0, 0, 538, 543, 3, 96, 48, 0, 539, 540, 5, 39, 0, 0, 540, 542, 3, 96, 48, 0, 541, 539, 1, 0, 0, 0, 542, 545, 1, 0, 0, 0, 543, 541, 1, 0, 0, 0, 543, 544, 1, 0, 0, 0, 544, 95, 1, 0, 0, 0, 545, 543, 1, 0, 0, 0, 546, 547, 3, 64, 32, 0, 547, 548, 5, 36, 0, 0, 548, 549, 3, 68, 34, 0, 549, 97, 1, 0, 0, 0, 550, 551, 7, 6, 0, 0, 551, 99, 1, 0, 0, 0, 552, 555, 3, 102, 51, 0, 553, 555, 3, 104, 52, 0, 554, 552, 1, 0, 0, 0, 554, 553, 1, 0, 0, 0, 555, 101, 1, 0, 0, 0, 556, 558, 7, 0, 0, 0, 557, 556, 1, 0, 0, 0, 557, 558, 1, 0, 0, 0, 558, 559, 1, 0, 0, 0, 559, 560, 5, 32, 0, 0, 560, 103, 1, 0, 0, 0, 561, 563, 7, 0, 0, 0, 562, 561, 1, 0, 0, 0, 562, 563, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 5, 31, 0, 0, 565, 105, 1, 0, 0, 0, 566, 567, 5, 30, 0, 0, 567, 107, 1, 0, 0, 0, 568, 569, 7, 7, 0, 0, 569, 109, 1, 0, 0, 0, 570, 571, 5, 5, 0, 0, 571, 572, 3, 112, 56, 0, 572, 111, 1, 0, 0, 0, 573, 574, 5, 70, 0, 0, 574, 575, 3, 2, 1, 0, 575, 576, 5, 71, 0, 0, 576, 113, 1, 0, 0, 0, 577, 578, 5, 13, 0, 0, 578, 579, 5, 105, 0, 0, 579, 115, 1, 0, 0, 0, 580, 581, 5, 3, 0, 0, 581, 584, 5, 95, 0, 0, 582, 583, 5, 93, 0, 0, 583, 585, 3, 60, 30, 0, 584, 582, 1, 0, 0, 0, 584, 585, 1, 0, 0, 0, 585, 595, 1, 0, 0, 0, 586, 587, 5, 94, 0, 0, 587, 592, 3, 118, 59, 0, 588, 589, 5, 39, 0, 0, 589, 591, 3, 118, 59, 0, 590, 588, 1, 0, 0, 0, 591, 594, 1, 0, 0, 0, 592, 590, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 596, 1, 0, 0, 0, 594, 592, 1, 0, 0, 0, 595, 586, 1, 0, 0, 0, 595, 596, 1, 0, 0, 0, 596, 117, 1, 0, 0, 0, 597, 598, 3, 60, 30, 0, 598, 599, 5, 36, 0, 0, 599, 601, 1, 0, 0, 0, 600, 597, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 3, 60, 30, 0, 603, 119, 1, 0, 0, 0, 604, 605, 5, 18, 0, 0, 605, 606, 3, 36, 18, 0, 606, 607, 5, 93, 0, 0, 607, 608, 3, 62, 31, 0, 608, 121, 1, 0, 0, 0, 609, 610, 5, 17, 0, 0, 610, 613, 3, 54, 27, 0, 611, 612, 5, 33, 0, 0, 612, 614, 3, 30, 15, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 617, 7, 8, 0, 0, 616, 615, 1, 0, 0, 0, 616, 617, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 619, 5, 20, 0, 0, 619, 620, 3, 126, 63, 0, 620, 621, 3, 128, 64, 0, 621, 125, 1, 0, 0, 0, 622, 625, 3, 64, 32, 0, 623, 624, 5, 89, 0, 0, 624, 626, 3, 64, 32, 0, 625, 623, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 127, 1, 0, 0, 0, 627, 628, 5, 93, 0, 0, 628, 633, 3, 130, 65, 0, 629, 630, 5, 39, 0, 0, 630, 632, 3, 130, 65, 0, 631, 629, 1, 0, 0, 0, 632, 635, 1, 0, 0, 0, 633, 631, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 129, 1, 0, 0, 0, 635, 633, 1, 0, 0, 0, 636, 637, 3, 16, 8, 0, 637, 131, 1, 0, 0, 0, 62, 143, 152, 172, 184, 193, 201, 206, 214, 216, 221, 228, 233, 238, 248, 254, 262, 264, 275, 282, 293, 296, 312, 318, 328, 332, 337, 347, 355, 368, 372, 376, 383, 387, 394, 400, 407, 415, 423, 431, 448, 459, 470, 475, 479, 484, 495, 500, 504, 518, 529, 543, 554, 557, 562, 584, 592, 595, 600, 613, 616, 625, 633] \ No newline at end of file +[4, 1, 128, 627, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7, 62, 2, 63, 7, 63, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 138, 8, 1, 10, 1, 12, 1, 141, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 149, 8, 2, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 3, 3, 169, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 181, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 188, 8, 5, 10, 5, 12, 5, 191, 9, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 198, 8, 5, 1, 5, 1, 5, 1, 5, 3, 5, 203, 8, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 1, 5, 5, 5, 211, 8, 5, 10, 5, 12, 5, 214, 9, 5, 1, 6, 1, 6, 3, 6, 218, 8, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 3, 6, 225, 8, 6, 1, 6, 1, 6, 1, 6, 3, 6, 230, 8, 6, 1, 7, 1, 7, 1, 7, 3, 7, 235, 8, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 3, 8, 245, 8, 8, 1, 9, 1, 9, 1, 9, 1, 9, 3, 9, 251, 8, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 1, 9, 5, 9, 259, 8, 9, 10, 9, 12, 9, 262, 9, 9, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 3, 10, 272, 8, 10, 1, 10, 1, 10, 1, 10, 5, 10, 277, 8, 10, 10, 10, 12, 10, 280, 9, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 5, 11, 288, 8, 11, 10, 11, 12, 11, 291, 9, 11, 3, 11, 293, 8, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 5, 15, 307, 8, 15, 10, 15, 12, 15, 310, 9, 15, 1, 16, 1, 16, 1, 16, 3, 16, 315, 8, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 323, 8, 17, 10, 17, 12, 17, 326, 9, 17, 1, 17, 3, 17, 329, 8, 17, 1, 18, 1, 18, 1, 18, 3, 18, 334, 8, 18, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 21, 5, 21, 346, 8, 21, 10, 21, 12, 21, 349, 9, 21, 1, 22, 1, 22, 1, 22, 1, 22, 5, 22, 355, 8, 22, 10, 22, 12, 22, 358, 9, 22, 1, 22, 3, 22, 361, 8, 22, 1, 22, 1, 22, 3, 22, 365, 8, 22, 1, 23, 1, 23, 1, 23, 1, 24, 1, 24, 3, 24, 372, 8, 24, 1, 24, 1, 24, 3, 24, 376, 8, 24, 1, 25, 1, 25, 1, 25, 5, 25, 381, 8, 25, 10, 25, 12, 25, 384, 9, 25, 1, 26, 1, 26, 1, 26, 3, 26, 389, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 394, 8, 27, 10, 27, 12, 27, 397, 9, 27, 1, 28, 1, 28, 1, 28, 5, 28, 402, 8, 28, 10, 28, 12, 28, 405, 9, 28, 1, 29, 1, 29, 1, 29, 5, 29, 410, 8, 29, 10, 29, 12, 29, 413, 9, 29, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 3, 31, 420, 8, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 435, 8, 32, 10, 32, 12, 32, 438, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 446, 8, 32, 10, 32, 12, 32, 449, 9, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 5, 32, 457, 8, 32, 10, 32, 12, 32, 460, 9, 32, 1, 32, 1, 32, 3, 32, 464, 8, 32, 1, 33, 1, 33, 3, 33, 468, 8, 33, 1, 34, 1, 34, 1, 34, 3, 34, 473, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 482, 8, 36, 10, 36, 12, 36, 485, 9, 36, 1, 37, 1, 37, 3, 37, 489, 8, 37, 1, 37, 1, 37, 3, 37, 493, 8, 37, 1, 38, 1, 38, 1, 38, 1, 39, 1, 39, 1, 39, 1, 40, 1, 40, 1, 40, 1, 40, 5, 40, 505, 8, 40, 10, 40, 12, 40, 508, 9, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 518, 8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 44, 1, 44, 1, 44, 1, 45, 1, 45, 1, 45, 5, 45, 530, 8, 45, 10, 45, 12, 45, 533, 9, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 48, 1, 48, 3, 48, 543, 8, 48, 1, 49, 3, 49, 546, 8, 49, 1, 49, 1, 49, 1, 50, 3, 50, 551, 8, 50, 1, 50, 1, 50, 1, 51, 1, 51, 1, 52, 1, 52, 1, 53, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 1, 54, 1, 55, 1, 55, 1, 55, 1, 56, 1, 56, 1, 56, 1, 56, 3, 56, 573, 8, 56, 1, 56, 1, 56, 1, 56, 1, 56, 5, 56, 579, 8, 56, 10, 56, 12, 56, 582, 9, 56, 3, 56, 584, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 589, 8, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 59, 1, 59, 1, 59, 1, 59, 3, 59, 602, 8, 59, 1, 60, 3, 60, 605, 8, 60, 1, 60, 1, 60, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 3, 61, 614, 8, 61, 1, 62, 1, 62, 1, 62, 1, 62, 5, 62, 620, 8, 62, 10, 62, 12, 62, 623, 9, 62, 1, 63, 1, 63, 1, 63, 0, 4, 2, 10, 18, 20, 64, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126, 0, 9, 1, 0, 64, 65, 1, 0, 66, 68, 2, 0, 30, 30, 81, 81, 1, 0, 72, 73, 2, 0, 35, 35, 40, 40, 2, 0, 43, 43, 46, 46, 2, 0, 42, 42, 56, 56, 2, 0, 57, 57, 59, 63, 1, 0, 22, 24, 654, 0, 128, 1, 0, 0, 0, 2, 131, 1, 0, 0, 0, 4, 148, 1, 0, 0, 0, 6, 168, 1, 0, 0, 0, 8, 170, 1, 0, 0, 0, 10, 202, 1, 0, 0, 0, 12, 229, 1, 0, 0, 0, 14, 231, 1, 0, 0, 0, 16, 244, 1, 0, 0, 0, 18, 250, 1, 0, 0, 0, 20, 271, 1, 0, 0, 0, 22, 281, 1, 0, 0, 0, 24, 296, 1, 0, 0, 0, 26, 298, 1, 0, 0, 0, 28, 300, 1, 0, 0, 0, 30, 303, 1, 0, 0, 0, 32, 314, 1, 0, 0, 0, 34, 318, 1, 0, 0, 0, 36, 333, 1, 0, 0, 0, 38, 337, 1, 0, 0, 0, 40, 339, 1, 0, 0, 0, 42, 341, 1, 0, 0, 0, 44, 350, 1, 0, 0, 0, 46, 366, 1, 0, 0, 0, 48, 369, 1, 0, 0, 0, 50, 377, 1, 0, 0, 0, 52, 385, 1, 0, 0, 0, 54, 390, 1, 0, 0, 0, 56, 398, 1, 0, 0, 0, 58, 406, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 419, 1, 0, 0, 0, 64, 463, 1, 0, 0, 0, 66, 467, 1, 0, 0, 0, 68, 472, 1, 0, 0, 0, 70, 474, 1, 0, 0, 0, 72, 477, 1, 0, 0, 0, 74, 486, 1, 0, 0, 0, 76, 494, 1, 0, 0, 0, 78, 497, 1, 0, 0, 0, 80, 500, 1, 0, 0, 0, 82, 509, 1, 0, 0, 0, 84, 513, 1, 0, 0, 0, 86, 519, 1, 0, 0, 0, 88, 523, 1, 0, 0, 0, 90, 526, 1, 0, 0, 0, 92, 534, 1, 0, 0, 0, 94, 538, 1, 0, 0, 0, 96, 542, 1, 0, 0, 0, 98, 545, 1, 0, 0, 0, 100, 550, 1, 0, 0, 0, 102, 554, 1, 0, 0, 0, 104, 556, 1, 0, 0, 0, 106, 558, 1, 0, 0, 0, 108, 561, 1, 0, 0, 0, 110, 565, 1, 0, 0, 0, 112, 568, 1, 0, 0, 0, 114, 588, 1, 0, 0, 0, 116, 592, 1, 0, 0, 0, 118, 597, 1, 0, 0, 0, 120, 604, 1, 0, 0, 0, 122, 610, 1, 0, 0, 0, 124, 615, 1, 0, 0, 0, 126, 624, 1, 0, 0, 0, 128, 129, 3, 2, 1, 0, 129, 130, 5, 0, 0, 1, 130, 1, 1, 0, 0, 0, 131, 132, 6, 1, -1, 0, 132, 133, 3, 4, 2, 0, 133, 139, 1, 0, 0, 0, 134, 135, 10, 1, 0, 0, 135, 136, 5, 29, 0, 0, 136, 138, 3, 6, 3, 0, 137, 134, 1, 0, 0, 0, 138, 141, 1, 0, 0, 0, 139, 137, 1, 0, 0, 0, 139, 140, 1, 0, 0, 0, 140, 3, 1, 0, 0, 0, 141, 139, 1, 0, 0, 0, 142, 149, 3, 106, 53, 0, 143, 149, 3, 34, 17, 0, 144, 149, 3, 28, 14, 0, 145, 149, 3, 110, 55, 0, 146, 147, 4, 2, 1, 0, 147, 149, 3, 44, 22, 0, 148, 142, 1, 0, 0, 0, 148, 143, 1, 0, 0, 0, 148, 144, 1, 0, 0, 0, 148, 145, 1, 0, 0, 0, 148, 146, 1, 0, 0, 0, 149, 5, 1, 0, 0, 0, 150, 169, 3, 46, 23, 0, 151, 169, 3, 8, 4, 0, 152, 169, 3, 76, 38, 0, 153, 169, 3, 70, 35, 0, 154, 169, 3, 48, 24, 0, 155, 169, 3, 72, 36, 0, 156, 169, 3, 78, 39, 0, 157, 169, 3, 80, 40, 0, 158, 169, 3, 84, 42, 0, 159, 169, 3, 86, 43, 0, 160, 169, 3, 112, 56, 0, 161, 169, 3, 88, 44, 0, 162, 163, 4, 3, 2, 0, 163, 169, 3, 118, 59, 0, 164, 165, 4, 3, 3, 0, 165, 169, 3, 116, 58, 0, 166, 167, 4, 3, 4, 0, 167, 169, 3, 120, 60, 0, 168, 150, 1, 0, 0, 0, 168, 151, 1, 0, 0, 0, 168, 152, 1, 0, 0, 0, 168, 153, 1, 0, 0, 0, 168, 154, 1, 0, 0, 0, 168, 155, 1, 0, 0, 0, 168, 156, 1, 0, 0, 0, 168, 157, 1, 0, 0, 0, 168, 158, 1, 0, 0, 0, 168, 159, 1, 0, 0, 0, 168, 160, 1, 0, 0, 0, 168, 161, 1, 0, 0, 0, 168, 162, 1, 0, 0, 0, 168, 164, 1, 0, 0, 0, 168, 166, 1, 0, 0, 0, 169, 7, 1, 0, 0, 0, 170, 171, 5, 16, 0, 0, 171, 172, 3, 10, 5, 0, 172, 9, 1, 0, 0, 0, 173, 174, 6, 5, -1, 0, 174, 175, 5, 49, 0, 0, 175, 203, 3, 10, 5, 8, 176, 203, 3, 16, 8, 0, 177, 203, 3, 12, 6, 0, 178, 180, 3, 16, 8, 0, 179, 181, 5, 49, 0, 0, 180, 179, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 182, 1, 0, 0, 0, 182, 183, 5, 44, 0, 0, 183, 184, 5, 48, 0, 0, 184, 189, 3, 16, 8, 0, 185, 186, 5, 39, 0, 0, 186, 188, 3, 16, 8, 0, 187, 185, 1, 0, 0, 0, 188, 191, 1, 0, 0, 0, 189, 187, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 192, 1, 0, 0, 0, 191, 189, 1, 0, 0, 0, 192, 193, 5, 55, 0, 0, 193, 203, 1, 0, 0, 0, 194, 195, 3, 16, 8, 0, 195, 197, 5, 45, 0, 0, 196, 198, 5, 49, 0, 0, 197, 196, 1, 0, 0, 0, 197, 198, 1, 0, 0, 0, 198, 199, 1, 0, 0, 0, 199, 200, 5, 50, 0, 0, 200, 203, 1, 0, 0, 0, 201, 203, 3, 14, 7, 0, 202, 173, 1, 0, 0, 0, 202, 176, 1, 0, 0, 0, 202, 177, 1, 0, 0, 0, 202, 178, 1, 0, 0, 0, 202, 194, 1, 0, 0, 0, 202, 201, 1, 0, 0, 0, 203, 212, 1, 0, 0, 0, 204, 205, 10, 5, 0, 0, 205, 206, 5, 34, 0, 0, 206, 211, 3, 10, 5, 6, 207, 208, 10, 4, 0, 0, 208, 209, 5, 52, 0, 0, 209, 211, 3, 10, 5, 5, 210, 204, 1, 0, 0, 0, 210, 207, 1, 0, 0, 0, 211, 214, 1, 0, 0, 0, 212, 210, 1, 0, 0, 0, 212, 213, 1, 0, 0, 0, 213, 11, 1, 0, 0, 0, 214, 212, 1, 0, 0, 0, 215, 217, 3, 16, 8, 0, 216, 218, 5, 49, 0, 0, 217, 216, 1, 0, 0, 0, 217, 218, 1, 0, 0, 0, 218, 219, 1, 0, 0, 0, 219, 220, 5, 47, 0, 0, 220, 221, 3, 102, 51, 0, 221, 230, 1, 0, 0, 0, 222, 224, 3, 16, 8, 0, 223, 225, 5, 49, 0, 0, 224, 223, 1, 0, 0, 0, 224, 225, 1, 0, 0, 0, 225, 226, 1, 0, 0, 0, 226, 227, 5, 54, 0, 0, 227, 228, 3, 102, 51, 0, 228, 230, 1, 0, 0, 0, 229, 215, 1, 0, 0, 0, 229, 222, 1, 0, 0, 0, 230, 13, 1, 0, 0, 0, 231, 234, 3, 54, 27, 0, 232, 233, 5, 37, 0, 0, 233, 235, 3, 26, 13, 0, 234, 232, 1, 0, 0, 0, 234, 235, 1, 0, 0, 0, 235, 236, 1, 0, 0, 0, 236, 237, 5, 38, 0, 0, 237, 238, 3, 64, 32, 0, 238, 15, 1, 0, 0, 0, 239, 245, 3, 18, 9, 0, 240, 241, 3, 18, 9, 0, 241, 242, 3, 104, 52, 0, 242, 243, 3, 18, 9, 0, 243, 245, 1, 0, 0, 0, 244, 239, 1, 0, 0, 0, 244, 240, 1, 0, 0, 0, 245, 17, 1, 0, 0, 0, 246, 247, 6, 9, -1, 0, 247, 251, 3, 20, 10, 0, 248, 249, 7, 0, 0, 0, 249, 251, 3, 18, 9, 3, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 251, 260, 1, 0, 0, 0, 252, 253, 10, 2, 0, 0, 253, 254, 7, 1, 0, 0, 254, 259, 3, 18, 9, 3, 255, 256, 10, 1, 0, 0, 256, 257, 7, 0, 0, 0, 257, 259, 3, 18, 9, 2, 258, 252, 1, 0, 0, 0, 258, 255, 1, 0, 0, 0, 259, 262, 1, 0, 0, 0, 260, 258, 1, 0, 0, 0, 260, 261, 1, 0, 0, 0, 261, 19, 1, 0, 0, 0, 262, 260, 1, 0, 0, 0, 263, 264, 6, 10, -1, 0, 264, 272, 3, 64, 32, 0, 265, 272, 3, 54, 27, 0, 266, 272, 3, 22, 11, 0, 267, 268, 5, 48, 0, 0, 268, 269, 3, 10, 5, 0, 269, 270, 5, 55, 0, 0, 270, 272, 1, 0, 0, 0, 271, 263, 1, 0, 0, 0, 271, 265, 1, 0, 0, 0, 271, 266, 1, 0, 0, 0, 271, 267, 1, 0, 0, 0, 272, 278, 1, 0, 0, 0, 273, 274, 10, 1, 0, 0, 274, 275, 5, 37, 0, 0, 275, 277, 3, 26, 13, 0, 276, 273, 1, 0, 0, 0, 277, 280, 1, 0, 0, 0, 278, 276, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 21, 1, 0, 0, 0, 280, 278, 1, 0, 0, 0, 281, 282, 3, 24, 12, 0, 282, 292, 5, 48, 0, 0, 283, 293, 5, 66, 0, 0, 284, 289, 3, 10, 5, 0, 285, 286, 5, 39, 0, 0, 286, 288, 3, 10, 5, 0, 287, 285, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289, 287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 293, 1, 0, 0, 0, 291, 289, 1, 0, 0, 0, 292, 283, 1, 0, 0, 0, 292, 284, 1, 0, 0, 0, 292, 293, 1, 0, 0, 0, 293, 294, 1, 0, 0, 0, 294, 295, 5, 55, 0, 0, 295, 23, 1, 0, 0, 0, 296, 297, 3, 68, 34, 0, 297, 25, 1, 0, 0, 0, 298, 299, 3, 60, 30, 0, 299, 27, 1, 0, 0, 0, 300, 301, 5, 12, 0, 0, 301, 302, 3, 30, 15, 0, 302, 29, 1, 0, 0, 0, 303, 308, 3, 32, 16, 0, 304, 305, 5, 39, 0, 0, 305, 307, 3, 32, 16, 0, 306, 304, 1, 0, 0, 0, 307, 310, 1, 0, 0, 0, 308, 306, 1, 0, 0, 0, 308, 309, 1, 0, 0, 0, 309, 31, 1, 0, 0, 0, 310, 308, 1, 0, 0, 0, 311, 312, 3, 54, 27, 0, 312, 313, 5, 36, 0, 0, 313, 315, 1, 0, 0, 0, 314, 311, 1, 0, 0, 0, 314, 315, 1, 0, 0, 0, 315, 316, 1, 0, 0, 0, 316, 317, 3, 10, 5, 0, 317, 33, 1, 0, 0, 0, 318, 319, 5, 6, 0, 0, 319, 324, 3, 36, 18, 0, 320, 321, 5, 39, 0, 0, 321, 323, 3, 36, 18, 0, 322, 320, 1, 0, 0, 0, 323, 326, 1, 0, 0, 0, 324, 322, 1, 0, 0, 0, 324, 325, 1, 0, 0, 0, 325, 328, 1, 0, 0, 0, 326, 324, 1, 0, 0, 0, 327, 329, 3, 42, 21, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 35, 1, 0, 0, 0, 330, 331, 3, 38, 19, 0, 331, 332, 5, 38, 0, 0, 332, 334, 1, 0, 0, 0, 333, 330, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 335, 1, 0, 0, 0, 335, 336, 3, 40, 20, 0, 336, 37, 1, 0, 0, 0, 337, 338, 5, 81, 0, 0, 338, 39, 1, 0, 0, 0, 339, 340, 7, 2, 0, 0, 340, 41, 1, 0, 0, 0, 341, 342, 5, 80, 0, 0, 342, 347, 5, 81, 0, 0, 343, 344, 5, 39, 0, 0, 344, 346, 5, 81, 0, 0, 345, 343, 1, 0, 0, 0, 346, 349, 1, 0, 0, 0, 347, 345, 1, 0, 0, 0, 347, 348, 1, 0, 0, 0, 348, 43, 1, 0, 0, 0, 349, 347, 1, 0, 0, 0, 350, 351, 5, 19, 0, 0, 351, 356, 3, 36, 18, 0, 352, 353, 5, 39, 0, 0, 353, 355, 3, 36, 18, 0, 354, 352, 1, 0, 0, 0, 355, 358, 1, 0, 0, 0, 356, 354, 1, 0, 0, 0, 356, 357, 1, 0, 0, 0, 357, 360, 1, 0, 0, 0, 358, 356, 1, 0, 0, 0, 359, 361, 3, 50, 25, 0, 360, 359, 1, 0, 0, 0, 360, 361, 1, 0, 0, 0, 361, 364, 1, 0, 0, 0, 362, 363, 5, 33, 0, 0, 363, 365, 3, 30, 15, 0, 364, 362, 1, 0, 0, 0, 364, 365, 1, 0, 0, 0, 365, 45, 1, 0, 0, 0, 366, 367, 5, 4, 0, 0, 367, 368, 3, 30, 15, 0, 368, 47, 1, 0, 0, 0, 369, 371, 5, 15, 0, 0, 370, 372, 3, 50, 25, 0, 371, 370, 1, 0, 0, 0, 371, 372, 1, 0, 0, 0, 372, 375, 1, 0, 0, 0, 373, 374, 5, 33, 0, 0, 374, 376, 3, 30, 15, 0, 375, 373, 1, 0, 0, 0, 375, 376, 1, 0, 0, 0, 376, 49, 1, 0, 0, 0, 377, 382, 3, 52, 26, 0, 378, 379, 5, 39, 0, 0, 379, 381, 3, 52, 26, 0, 380, 378, 1, 0, 0, 0, 381, 384, 1, 0, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 51, 1, 0, 0, 0, 384, 382, 1, 0, 0, 0, 385, 388, 3, 32, 16, 0, 386, 387, 5, 16, 0, 0, 387, 389, 3, 10, 5, 0, 388, 386, 1, 0, 0, 0, 388, 389, 1, 0, 0, 0, 389, 53, 1, 0, 0, 0, 390, 395, 3, 68, 34, 0, 391, 392, 5, 41, 0, 0, 392, 394, 3, 68, 34, 0, 393, 391, 1, 0, 0, 0, 394, 397, 1, 0, 0, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 55, 1, 0, 0, 0, 397, 395, 1, 0, 0, 0, 398, 403, 3, 62, 31, 0, 399, 400, 5, 41, 0, 0, 400, 402, 3, 62, 31, 0, 401, 399, 1, 0, 0, 0, 402, 405, 1, 0, 0, 0, 403, 401, 1, 0, 0, 0, 403, 404, 1, 0, 0, 0, 404, 57, 1, 0, 0, 0, 405, 403, 1, 0, 0, 0, 406, 411, 3, 56, 28, 0, 407, 408, 5, 39, 0, 0, 408, 410, 3, 56, 28, 0, 409, 407, 1, 0, 0, 0, 410, 413, 1, 0, 0, 0, 411, 409, 1, 0, 0, 0, 411, 412, 1, 0, 0, 0, 412, 59, 1, 0, 0, 0, 413, 411, 1, 0, 0, 0, 414, 415, 7, 3, 0, 0, 415, 61, 1, 0, 0, 0, 416, 420, 5, 85, 0, 0, 417, 418, 4, 31, 10, 0, 418, 420, 3, 66, 33, 0, 419, 416, 1, 0, 0, 0, 419, 417, 1, 0, 0, 0, 420, 63, 1, 0, 0, 0, 421, 464, 5, 50, 0, 0, 422, 423, 3, 100, 50, 0, 423, 424, 5, 72, 0, 0, 424, 464, 1, 0, 0, 0, 425, 464, 3, 98, 49, 0, 426, 464, 3, 100, 50, 0, 427, 464, 3, 94, 47, 0, 428, 464, 3, 66, 33, 0, 429, 464, 3, 102, 51, 0, 430, 431, 5, 70, 0, 0, 431, 436, 3, 96, 48, 0, 432, 433, 5, 39, 0, 0, 433, 435, 3, 96, 48, 0, 434, 432, 1, 0, 0, 0, 435, 438, 1, 0, 0, 0, 436, 434, 1, 0, 0, 0, 436, 437, 1, 0, 0, 0, 437, 439, 1, 0, 0, 0, 438, 436, 1, 0, 0, 0, 439, 440, 5, 71, 0, 0, 440, 464, 1, 0, 0, 0, 441, 442, 5, 70, 0, 0, 442, 447, 3, 94, 47, 0, 443, 444, 5, 39, 0, 0, 444, 446, 3, 94, 47, 0, 445, 443, 1, 0, 0, 0, 446, 449, 1, 0, 0, 0, 447, 445, 1, 0, 0, 0, 447, 448, 1, 0, 0, 0, 448, 450, 1, 0, 0, 0, 449, 447, 1, 0, 0, 0, 450, 451, 5, 71, 0, 0, 451, 464, 1, 0, 0, 0, 452, 453, 5, 70, 0, 0, 453, 458, 3, 102, 51, 0, 454, 455, 5, 39, 0, 0, 455, 457, 3, 102, 51, 0, 456, 454, 1, 0, 0, 0, 457, 460, 1, 0, 0, 0, 458, 456, 1, 0, 0, 0, 458, 459, 1, 0, 0, 0, 459, 461, 1, 0, 0, 0, 460, 458, 1, 0, 0, 0, 461, 462, 5, 71, 0, 0, 462, 464, 1, 0, 0, 0, 463, 421, 1, 0, 0, 0, 463, 422, 1, 0, 0, 0, 463, 425, 1, 0, 0, 0, 463, 426, 1, 0, 0, 0, 463, 427, 1, 0, 0, 0, 463, 428, 1, 0, 0, 0, 463, 429, 1, 0, 0, 0, 463, 430, 1, 0, 0, 0, 463, 441, 1, 0, 0, 0, 463, 452, 1, 0, 0, 0, 464, 65, 1, 0, 0, 0, 465, 468, 5, 53, 0, 0, 466, 468, 5, 69, 0, 0, 467, 465, 1, 0, 0, 0, 467, 466, 1, 0, 0, 0, 468, 67, 1, 0, 0, 0, 469, 473, 3, 60, 30, 0, 470, 471, 4, 34, 11, 0, 471, 473, 3, 66, 33, 0, 472, 469, 1, 0, 0, 0, 472, 470, 1, 0, 0, 0, 473, 69, 1, 0, 0, 0, 474, 475, 5, 9, 0, 0, 475, 476, 5, 31, 0, 0, 476, 71, 1, 0, 0, 0, 477, 478, 5, 14, 0, 0, 478, 483, 3, 74, 37, 0, 479, 480, 5, 39, 0, 0, 480, 482, 3, 74, 37, 0, 481, 479, 1, 0, 0, 0, 482, 485, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 483, 484, 1, 0, 0, 0, 484, 73, 1, 0, 0, 0, 485, 483, 1, 0, 0, 0, 486, 488, 3, 10, 5, 0, 487, 489, 7, 4, 0, 0, 488, 487, 1, 0, 0, 0, 488, 489, 1, 0, 0, 0, 489, 492, 1, 0, 0, 0, 490, 491, 5, 51, 0, 0, 491, 493, 7, 5, 0, 0, 492, 490, 1, 0, 0, 0, 492, 493, 1, 0, 0, 0, 493, 75, 1, 0, 0, 0, 494, 495, 5, 8, 0, 0, 495, 496, 3, 58, 29, 0, 496, 77, 1, 0, 0, 0, 497, 498, 5, 2, 0, 0, 498, 499, 3, 58, 29, 0, 499, 79, 1, 0, 0, 0, 500, 501, 5, 11, 0, 0, 501, 506, 3, 82, 41, 0, 502, 503, 5, 39, 0, 0, 503, 505, 3, 82, 41, 0, 504, 502, 1, 0, 0, 0, 505, 508, 1, 0, 0, 0, 506, 504, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 81, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 509, 510, 3, 56, 28, 0, 510, 511, 5, 89, 0, 0, 511, 512, 3, 56, 28, 0, 512, 83, 1, 0, 0, 0, 513, 514, 5, 1, 0, 0, 514, 515, 3, 20, 10, 0, 515, 517, 3, 102, 51, 0, 516, 518, 3, 90, 45, 0, 517, 516, 1, 0, 0, 0, 517, 518, 1, 0, 0, 0, 518, 85, 1, 0, 0, 0, 519, 520, 5, 7, 0, 0, 520, 521, 3, 20, 10, 0, 521, 522, 3, 102, 51, 0, 522, 87, 1, 0, 0, 0, 523, 524, 5, 10, 0, 0, 524, 525, 3, 54, 27, 0, 525, 89, 1, 0, 0, 0, 526, 531, 3, 92, 46, 0, 527, 528, 5, 39, 0, 0, 528, 530, 3, 92, 46, 0, 529, 527, 1, 0, 0, 0, 530, 533, 1, 0, 0, 0, 531, 529, 1, 0, 0, 0, 531, 532, 1, 0, 0, 0, 532, 91, 1, 0, 0, 0, 533, 531, 1, 0, 0, 0, 534, 535, 3, 60, 30, 0, 535, 536, 5, 36, 0, 0, 536, 537, 3, 64, 32, 0, 537, 93, 1, 0, 0, 0, 538, 539, 7, 6, 0, 0, 539, 95, 1, 0, 0, 0, 540, 543, 3, 98, 49, 0, 541, 543, 3, 100, 50, 0, 542, 540, 1, 0, 0, 0, 542, 541, 1, 0, 0, 0, 543, 97, 1, 0, 0, 0, 544, 546, 7, 0, 0, 0, 545, 544, 1, 0, 0, 0, 545, 546, 1, 0, 0, 0, 546, 547, 1, 0, 0, 0, 547, 548, 5, 32, 0, 0, 548, 99, 1, 0, 0, 0, 549, 551, 7, 0, 0, 0, 550, 549, 1, 0, 0, 0, 550, 551, 1, 0, 0, 0, 551, 552, 1, 0, 0, 0, 552, 553, 5, 31, 0, 0, 553, 101, 1, 0, 0, 0, 554, 555, 5, 30, 0, 0, 555, 103, 1, 0, 0, 0, 556, 557, 7, 7, 0, 0, 557, 105, 1, 0, 0, 0, 558, 559, 5, 5, 0, 0, 559, 560, 3, 108, 54, 0, 560, 107, 1, 0, 0, 0, 561, 562, 5, 70, 0, 0, 562, 563, 3, 2, 1, 0, 563, 564, 5, 71, 0, 0, 564, 109, 1, 0, 0, 0, 565, 566, 5, 13, 0, 0, 566, 567, 5, 105, 0, 0, 567, 111, 1, 0, 0, 0, 568, 569, 5, 3, 0, 0, 569, 572, 5, 95, 0, 0, 570, 571, 5, 93, 0, 0, 571, 573, 3, 56, 28, 0, 572, 570, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 583, 1, 0, 0, 0, 574, 575, 5, 94, 0, 0, 575, 580, 3, 114, 57, 0, 576, 577, 5, 39, 0, 0, 577, 579, 3, 114, 57, 0, 578, 576, 1, 0, 0, 0, 579, 582, 1, 0, 0, 0, 580, 578, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 584, 1, 0, 0, 0, 582, 580, 1, 0, 0, 0, 583, 574, 1, 0, 0, 0, 583, 584, 1, 0, 0, 0, 584, 113, 1, 0, 0, 0, 585, 586, 3, 56, 28, 0, 586, 587, 5, 36, 0, 0, 587, 589, 1, 0, 0, 0, 588, 585, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 3, 56, 28, 0, 591, 115, 1, 0, 0, 0, 592, 593, 5, 18, 0, 0, 593, 594, 3, 36, 18, 0, 594, 595, 5, 93, 0, 0, 595, 596, 3, 58, 29, 0, 596, 117, 1, 0, 0, 0, 597, 598, 5, 17, 0, 0, 598, 601, 3, 50, 25, 0, 599, 600, 5, 33, 0, 0, 600, 602, 3, 30, 15, 0, 601, 599, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 119, 1, 0, 0, 0, 603, 605, 7, 8, 0, 0, 604, 603, 1, 0, 0, 0, 604, 605, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 5, 20, 0, 0, 607, 608, 3, 122, 61, 0, 608, 609, 3, 124, 62, 0, 609, 121, 1, 0, 0, 0, 610, 613, 3, 60, 30, 0, 611, 612, 5, 89, 0, 0, 612, 614, 3, 60, 30, 0, 613, 611, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 123, 1, 0, 0, 0, 615, 616, 5, 93, 0, 0, 616, 621, 3, 126, 63, 0, 617, 618, 5, 39, 0, 0, 618, 620, 3, 126, 63, 0, 619, 617, 1, 0, 0, 0, 620, 623, 1, 0, 0, 0, 621, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 125, 1, 0, 0, 0, 623, 621, 1, 0, 0, 0, 624, 625, 3, 16, 8, 0, 625, 127, 1, 0, 0, 0, 61, 139, 148, 168, 180, 189, 197, 202, 210, 212, 217, 224, 229, 234, 244, 250, 258, 260, 271, 278, 289, 292, 308, 314, 324, 328, 333, 347, 356, 360, 364, 371, 375, 382, 388, 395, 403, 411, 419, 436, 447, 458, 463, 467, 472, 483, 488, 492, 506, 517, 531, 542, 545, 550, 572, 580, 583, 588, 601, 604, 613, 621] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java index a56035364641d..e272dc7f477a4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser.java @@ -59,35 +59,34 @@ public class EsqlBaseParser extends ParserConfig { RULE_primaryExpression = 10, RULE_functionExpression = 11, RULE_functionName = 12, RULE_dataType = 13, RULE_rowCommand = 14, RULE_fields = 15, RULE_field = 16, RULE_fromCommand = 17, RULE_indexPattern = 18, RULE_clusterString = 19, - RULE_indexString = 20, RULE_metadata = 21, RULE_metadataOption = 22, RULE_deprecated_metadata = 23, - RULE_metricsCommand = 24, RULE_evalCommand = 25, RULE_statsCommand = 26, - RULE_aggFields = 27, RULE_aggField = 28, RULE_qualifiedName = 29, RULE_qualifiedNamePattern = 30, - RULE_qualifiedNamePatterns = 31, RULE_identifier = 32, RULE_identifierPattern = 33, - RULE_constant = 34, RULE_parameter = 35, RULE_identifierOrParameter = 36, - RULE_limitCommand = 37, RULE_sortCommand = 38, RULE_orderExpression = 39, - RULE_keepCommand = 40, RULE_dropCommand = 41, RULE_renameCommand = 42, - RULE_renameClause = 43, RULE_dissectCommand = 44, RULE_grokCommand = 45, - RULE_mvExpandCommand = 46, RULE_commandOptions = 47, RULE_commandOption = 48, - RULE_booleanValue = 49, RULE_numericValue = 50, RULE_decimalValue = 51, - RULE_integerValue = 52, RULE_string = 53, RULE_comparisonOperator = 54, - RULE_explainCommand = 55, RULE_subqueryExpression = 56, RULE_showCommand = 57, - RULE_enrichCommand = 58, RULE_enrichWithClause = 59, RULE_lookupCommand = 60, - RULE_inlinestatsCommand = 61, RULE_joinCommand = 62, RULE_joinTarget = 63, - RULE_joinCondition = 64, RULE_joinPredicate = 65; + RULE_indexString = 20, RULE_metadata = 21, RULE_metricsCommand = 22, RULE_evalCommand = 23, + RULE_statsCommand = 24, RULE_aggFields = 25, RULE_aggField = 26, RULE_qualifiedName = 27, + RULE_qualifiedNamePattern = 28, RULE_qualifiedNamePatterns = 29, RULE_identifier = 30, + RULE_identifierPattern = 31, RULE_constant = 32, RULE_parameter = 33, + RULE_identifierOrParameter = 34, RULE_limitCommand = 35, RULE_sortCommand = 36, + RULE_orderExpression = 37, RULE_keepCommand = 38, RULE_dropCommand = 39, + RULE_renameCommand = 40, RULE_renameClause = 41, RULE_dissectCommand = 42, + RULE_grokCommand = 43, RULE_mvExpandCommand = 44, RULE_commandOptions = 45, + RULE_commandOption = 46, RULE_booleanValue = 47, RULE_numericValue = 48, + RULE_decimalValue = 49, RULE_integerValue = 50, RULE_string = 51, RULE_comparisonOperator = 52, + RULE_explainCommand = 53, RULE_subqueryExpression = 54, RULE_showCommand = 55, + RULE_enrichCommand = 56, RULE_enrichWithClause = 57, RULE_lookupCommand = 58, + RULE_inlinestatsCommand = 59, RULE_joinCommand = 60, RULE_joinTarget = 61, + RULE_joinCondition = 62, RULE_joinPredicate = 63; private static String[] makeRuleNames() { return new String[] { "singleStatement", "query", "sourceCommand", "processingCommand", "whereCommand", "booleanExpression", "regexBooleanExpression", "matchBooleanExpression", "valueExpression", "operatorExpression", "primaryExpression", "functionExpression", "functionName", "dataType", "rowCommand", "fields", "field", "fromCommand", - "indexPattern", "clusterString", "indexString", "metadata", "metadataOption", - "deprecated_metadata", "metricsCommand", "evalCommand", "statsCommand", - "aggFields", "aggField", "qualifiedName", "qualifiedNamePattern", "qualifiedNamePatterns", - "identifier", "identifierPattern", "constant", "parameter", "identifierOrParameter", - "limitCommand", "sortCommand", "orderExpression", "keepCommand", "dropCommand", - "renameCommand", "renameClause", "dissectCommand", "grokCommand", "mvExpandCommand", - "commandOptions", "commandOption", "booleanValue", "numericValue", "decimalValue", - "integerValue", "string", "comparisonOperator", "explainCommand", "subqueryExpression", + "indexPattern", "clusterString", "indexString", "metadata", "metricsCommand", + "evalCommand", "statsCommand", "aggFields", "aggField", "qualifiedName", + "qualifiedNamePattern", "qualifiedNamePatterns", "identifier", "identifierPattern", + "constant", "parameter", "identifierOrParameter", "limitCommand", "sortCommand", + "orderExpression", "keepCommand", "dropCommand", "renameCommand", "renameClause", + "dissectCommand", "grokCommand", "mvExpandCommand", "commandOptions", + "commandOption", "booleanValue", "numericValue", "decimalValue", "integerValue", + "string", "comparisonOperator", "explainCommand", "subqueryExpression", "showCommand", "enrichCommand", "enrichWithClause", "lookupCommand", "inlinestatsCommand", "joinCommand", "joinTarget", "joinCondition", "joinPredicate" }; @@ -226,9 +225,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(132); + setState(128); query(0); - setState(133); + setState(129); match(EOF); } } @@ -324,11 +323,11 @@ private QueryContext query(int _p) throws RecognitionException { _ctx = _localctx; _prevctx = _localctx; - setState(136); + setState(132); sourceCommand(); } _ctx.stop = _input.LT(-1); - setState(143); + setState(139); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -339,16 +338,16 @@ private QueryContext query(int _p) throws RecognitionException { { _localctx = new CompositeQueryContext(new QueryContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_query); - setState(138); + setState(134); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(139); + setState(135); match(PIPE); - setState(140); + setState(136); processingCommand(); } } } - setState(145); + setState(141); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,0,_ctx); } @@ -406,43 +405,43 @@ public final SourceCommandContext sourceCommand() throws RecognitionException { SourceCommandContext _localctx = new SourceCommandContext(_ctx, getState()); enterRule(_localctx, 4, RULE_sourceCommand); try { - setState(152); + setState(148); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,1,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(146); + setState(142); explainCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(147); + setState(143); fromCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(148); + setState(144); rowCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(149); + setState(145); showCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(150); + setState(146); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(151); + setState(147); metricsCommand(); } break; @@ -530,117 +529,117 @@ public final ProcessingCommandContext processingCommand() throws RecognitionExce ProcessingCommandContext _localctx = new ProcessingCommandContext(_ctx, getState()); enterRule(_localctx, 6, RULE_processingCommand); try { - setState(172); + setState(168); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(154); + setState(150); evalCommand(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(155); + setState(151); whereCommand(); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(156); + setState(152); keepCommand(); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(157); + setState(153); limitCommand(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(158); + setState(154); statsCommand(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(159); + setState(155); sortCommand(); } break; case 7: enterOuterAlt(_localctx, 7); { - setState(160); + setState(156); dropCommand(); } break; case 8: enterOuterAlt(_localctx, 8); { - setState(161); + setState(157); renameCommand(); } break; case 9: enterOuterAlt(_localctx, 9); { - setState(162); + setState(158); dissectCommand(); } break; case 10: enterOuterAlt(_localctx, 10); { - setState(163); + setState(159); grokCommand(); } break; case 11: enterOuterAlt(_localctx, 11); { - setState(164); + setState(160); enrichCommand(); } break; case 12: enterOuterAlt(_localctx, 12); { - setState(165); + setState(161); mvExpandCommand(); } break; case 13: enterOuterAlt(_localctx, 13); { - setState(166); + setState(162); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(167); + setState(163); inlinestatsCommand(); } break; case 14: enterOuterAlt(_localctx, 14); { - setState(168); + setState(164); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(169); + setState(165); lookupCommand(); } break; case 15: enterOuterAlt(_localctx, 15); { - setState(170); + setState(166); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(171); + setState(167); joinCommand(); } break; @@ -689,9 +688,9 @@ public final WhereCommandContext whereCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(174); + setState(170); match(WHERE); - setState(175); + setState(171); booleanExpression(0); } } @@ -907,7 +906,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(206); + setState(202); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,6,_ctx) ) { case 1: @@ -916,9 +915,9 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(178); + setState(174); match(NOT); - setState(179); + setState(175); booleanExpression(8); } break; @@ -927,7 +926,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(180); + setState(176); valueExpression(); } break; @@ -936,7 +935,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new RegexExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(181); + setState(177); regexBooleanExpression(); } break; @@ -945,41 +944,41 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalInContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(182); + setState(178); valueExpression(); - setState(184); + setState(180); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(183); + setState(179); match(NOT); } } - setState(186); + setState(182); match(IN); - setState(187); + setState(183); match(LP); - setState(188); + setState(184); valueExpression(); - setState(193); + setState(189); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(189); + setState(185); match(COMMA); - setState(190); + setState(186); valueExpression(); } } - setState(195); + setState(191); _errHandler.sync(this); _la = _input.LA(1); } - setState(196); + setState(192); match(RP); } break; @@ -988,21 +987,21 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new IsNullContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(198); + setState(194); valueExpression(); - setState(199); + setState(195); match(IS); - setState(201); + setState(197); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(200); + setState(196); match(NOT); } } - setState(203); + setState(199); match(NULL); } break; @@ -1011,13 +1010,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(205); + setState(201); matchBooleanExpression(); } break; } _ctx.stop = _input.LT(-1); - setState(216); + setState(212); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1025,7 +1024,7 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(214); + setState(210); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,7,_ctx) ) { case 1: @@ -1033,11 +1032,11 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(208); + setState(204); if (!(precpred(_ctx, 5))) throw new FailedPredicateException(this, "precpred(_ctx, 5)"); - setState(209); + setState(205); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(210); + setState(206); ((LogicalBinaryContext)_localctx).right = booleanExpression(6); } break; @@ -1046,18 +1045,18 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(211); + setState(207); if (!(precpred(_ctx, 4))) throw new FailedPredicateException(this, "precpred(_ctx, 4)"); - setState(212); + setState(208); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(213); + setState(209); ((LogicalBinaryContext)_localctx).right = booleanExpression(5); } break; } } } - setState(218); + setState(214); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,8,_ctx); } @@ -1112,48 +1111,48 @@ public final RegexBooleanExpressionContext regexBooleanExpression() throws Recog enterRule(_localctx, 12, RULE_regexBooleanExpression); int _la; try { - setState(233); + setState(229); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,11,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(219); + setState(215); valueExpression(); - setState(221); + setState(217); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(220); + setState(216); match(NOT); } } - setState(223); + setState(219); ((RegexBooleanExpressionContext)_localctx).kind = match(LIKE); - setState(224); + setState(220); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(226); + setState(222); valueExpression(); - setState(228); + setState(224); _errHandler.sync(this); _la = _input.LA(1); if (_la==NOT) { { - setState(227); + setState(223); match(NOT); } } - setState(230); + setState(226); ((RegexBooleanExpressionContext)_localctx).kind = match(RLIKE); - setState(231); + setState(227); ((RegexBooleanExpressionContext)_localctx).pattern = string(); } break; @@ -1213,23 +1212,23 @@ public final MatchBooleanExpressionContext matchBooleanExpression() throws Recog try { enterOuterAlt(_localctx, 1); { - setState(235); + setState(231); ((MatchBooleanExpressionContext)_localctx).fieldExp = qualifiedName(); - setState(238); + setState(234); _errHandler.sync(this); _la = _input.LA(1); if (_la==CAST_OP) { { - setState(236); + setState(232); match(CAST_OP); - setState(237); + setState(233); ((MatchBooleanExpressionContext)_localctx).fieldType = dataType(); } } - setState(240); + setState(236); match(COLON); - setState(241); + setState(237); ((MatchBooleanExpressionContext)_localctx).matchQuery = constant(); } } @@ -1313,14 +1312,14 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, getState()); enterRule(_localctx, 16, RULE_valueExpression); try { - setState(248); + setState(244); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,13,_ctx) ) { case 1: _localctx = new ValueExpressionDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(243); + setState(239); operatorExpression(0); } break; @@ -1328,11 +1327,11 @@ public final ValueExpressionContext valueExpression() throws RecognitionExceptio _localctx = new ComparisonContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(244); + setState(240); ((ComparisonContext)_localctx).left = operatorExpression(0); - setState(245); + setState(241); comparisonOperator(); - setState(246); + setState(242); ((ComparisonContext)_localctx).right = operatorExpression(0); } break; @@ -1457,7 +1456,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE int _alt; enterOuterAlt(_localctx, 1); { - setState(254); + setState(250); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,14,_ctx) ) { case 1: @@ -1466,7 +1465,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _ctx = _localctx; _prevctx = _localctx; - setState(251); + setState(247); primaryExpression(0); } break; @@ -1475,7 +1474,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(252); + setState(248); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1486,13 +1485,13 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(253); + setState(249); operatorExpression(3); } break; } _ctx.stop = _input.LT(-1); - setState(264); + setState(260); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1500,7 +1499,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(262); + setState(258); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,15,_ctx) ) { case 1: @@ -1508,9 +1507,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(256); + setState(252); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(257); + setState(253); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(((((_la - 66)) & ~0x3f) == 0 && ((1L << (_la - 66)) & 7L) != 0)) ) { @@ -1521,7 +1520,7 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(258); + setState(254); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(3); } break; @@ -1530,9 +1529,9 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _localctx = new ArithmeticBinaryContext(new OperatorExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_operatorExpression); - setState(259); + setState(255); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(260); + setState(256); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -1543,14 +1542,14 @@ private OperatorExpressionContext operatorExpression(int _p) throws RecognitionE _errHandler.reportMatch(this); consume(); } - setState(261); + setState(257); ((ArithmeticBinaryContext)_localctx).right = operatorExpression(2); } break; } } } - setState(266); + setState(262); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,16,_ctx); } @@ -1708,7 +1707,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc int _alt; enterOuterAlt(_localctx, 1); { - setState(275); + setState(271); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,17,_ctx) ) { case 1: @@ -1717,7 +1716,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _ctx = _localctx; _prevctx = _localctx; - setState(268); + setState(264); constant(); } break; @@ -1726,7 +1725,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new DereferenceContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(269); + setState(265); qualifiedName(); } break; @@ -1735,7 +1734,7 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new FunctionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(270); + setState(266); functionExpression(); } break; @@ -1744,17 +1743,17 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc _localctx = new ParenthesizedExpressionContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(271); + setState(267); match(LP); - setState(272); + setState(268); booleanExpression(0); - setState(273); + setState(269); match(RP); } break; } _ctx.stop = _input.LT(-1); - setState(282); + setState(278); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { @@ -1765,16 +1764,16 @@ private PrimaryExpressionContext primaryExpression(int _p) throws RecognitionExc { _localctx = new InlineCastContext(new PrimaryExpressionContext(_parentctx, _parentState)); pushNewRecursionContext(_localctx, _startState, RULE_primaryExpression); - setState(277); + setState(273); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(278); + setState(274); match(CAST_OP); - setState(279); + setState(275); dataType(); } } } - setState(284); + setState(280); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,18,_ctx); } @@ -1836,37 +1835,37 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx try { enterOuterAlt(_localctx, 1); { - setState(285); + setState(281); functionName(); - setState(286); + setState(282); match(LP); - setState(296); + setState(292); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,20,_ctx) ) { case 1: { - setState(287); + setState(283); match(ASTERISK); } break; case 2: { { - setState(288); + setState(284); booleanExpression(0); - setState(293); + setState(289); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(289); + setState(285); match(COMMA); - setState(290); + setState(286); booleanExpression(0); } } - setState(295); + setState(291); _errHandler.sync(this); _la = _input.LA(1); } @@ -1874,7 +1873,7 @@ public final FunctionExpressionContext functionExpression() throws RecognitionEx } break; } - setState(298); + setState(294); match(RP); } } @@ -1920,7 +1919,7 @@ public final FunctionNameContext functionName() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(300); + setState(296); identifierOrParameter(); } } @@ -1978,7 +1977,7 @@ public final DataTypeContext dataType() throws RecognitionException { _localctx = new ToDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(302); + setState(298); identifier(); } } @@ -2025,9 +2024,9 @@ public final RowCommandContext rowCommand() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(304); + setState(300); match(ROW); - setState(305); + setState(301); fields(); } } @@ -2081,23 +2080,23 @@ public final FieldsContext fields() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(307); + setState(303); field(); - setState(312); + setState(308); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(308); + setState(304); match(COMMA); - setState(309); + setState(305); field(); } } } - setState(314); + setState(310); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,21,_ctx); } @@ -2149,19 +2148,19 @@ public final FieldContext field() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(318); + setState(314); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,22,_ctx) ) { case 1: { - setState(315); + setState(311); qualifiedName(); - setState(316); + setState(312); match(ASSIGN); } break; } - setState(320); + setState(316); booleanExpression(0); } } @@ -2219,34 +2218,34 @@ public final FromCommandContext fromCommand() throws RecognitionException { int _alt; enterOuterAlt(_localctx, 1); { - setState(322); + setState(318); match(FROM); - setState(323); + setState(319); indexPattern(); - setState(328); + setState(324); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(324); + setState(320); match(COMMA); - setState(325); + setState(321); indexPattern(); } } } - setState(330); + setState(326); _errHandler.sync(this); _alt = getInterpreter().adaptivePredict(_input,23,_ctx); } - setState(332); + setState(328); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,24,_ctx) ) { case 1: { - setState(331); + setState(327); metadata(); } break; @@ -2299,19 +2298,19 @@ public final IndexPatternContext indexPattern() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(337); + setState(333); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,25,_ctx) ) { case 1: { - setState(334); + setState(330); clusterString(); - setState(335); + setState(331); match(COLON); } break; } - setState(339); + setState(335); indexString(); } } @@ -2355,7 +2354,7 @@ public final ClusterStringContext clusterString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(341); + setState(337); match(UNQUOTED_SOURCE); } } @@ -2401,7 +2400,7 @@ public final IndexStringContext indexString() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(343); + setState(339); _la = _input.LA(1); if ( !(_la==QUOTED_STRING || _la==UNQUOTED_SOURCE) ) { _errHandler.recoverInline(this); @@ -2426,11 +2425,14 @@ public final IndexStringContext indexString() throws RecognitionException { @SuppressWarnings("CheckReturnValue") public static class MetadataContext extends ParserRuleContext { - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); + public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } + public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } + public TerminalNode UNQUOTED_SOURCE(int i) { + return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); } - public Deprecated_metadataContext deprecated_metadata() { - return getRuleContext(Deprecated_metadataContext.class,0); + public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } + public TerminalNode COMMA(int i) { + return getToken(EsqlBaseParser.COMMA, i); } @SuppressWarnings("this-escape") public MetadataContext(ParserRuleContext parent, int invokingState) { @@ -2455,98 +2457,31 @@ public T accept(ParseTreeVisitor visitor) { public final MetadataContext metadata() throws RecognitionException { MetadataContext _localctx = new MetadataContext(_ctx, getState()); enterRule(_localctx, 42, RULE_metadata); - try { - setState(347); - _errHandler.sync(this); - switch (_input.LA(1)) { - case METADATA: - enterOuterAlt(_localctx, 1); - { - setState(345); - metadataOption(); - } - break; - case OPENING_BRACKET: - enterOuterAlt(_localctx, 2); - { - setState(346); - deprecated_metadata(); - } - break; - default: - throw new NoViableAltException(this); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - - @SuppressWarnings("CheckReturnValue") - public static class MetadataOptionContext extends ParserRuleContext { - public TerminalNode METADATA() { return getToken(EsqlBaseParser.METADATA, 0); } - public List UNQUOTED_SOURCE() { return getTokens(EsqlBaseParser.UNQUOTED_SOURCE); } - public TerminalNode UNQUOTED_SOURCE(int i) { - return getToken(EsqlBaseParser.UNQUOTED_SOURCE, i); - } - public List COMMA() { return getTokens(EsqlBaseParser.COMMA); } - public TerminalNode COMMA(int i) { - return getToken(EsqlBaseParser.COMMA, i); - } - @SuppressWarnings("this-escape") - public MetadataOptionContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_metadataOption; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterMetadataOption(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitMetadataOption(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitMetadataOption(this); - else return visitor.visitChildren(this); - } - } - - public final MetadataOptionContext metadataOption() throws RecognitionException { - MetadataOptionContext _localctx = new MetadataOptionContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_metadataOption); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(349); + setState(341); match(METADATA); - setState(350); + setState(342); match(UNQUOTED_SOURCE); - setState(355); + setState(347); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(351); + setState(343); match(COMMA); - setState(352); + setState(344); match(UNQUOTED_SOURCE); } } } - setState(357); + setState(349); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,27,_ctx); + _alt = getInterpreter().adaptivePredict(_input,26,_ctx); } } } @@ -2561,57 +2496,6 @@ public final MetadataOptionContext metadataOption() throws RecognitionException return _localctx; } - @SuppressWarnings("CheckReturnValue") - public static class Deprecated_metadataContext extends ParserRuleContext { - public TerminalNode OPENING_BRACKET() { return getToken(EsqlBaseParser.OPENING_BRACKET, 0); } - public MetadataOptionContext metadataOption() { - return getRuleContext(MetadataOptionContext.class,0); - } - public TerminalNode CLOSING_BRACKET() { return getToken(EsqlBaseParser.CLOSING_BRACKET, 0); } - public Deprecated_metadataContext(ParserRuleContext parent, int invokingState) { - super(parent, invokingState); - } - @Override public int getRuleIndex() { return RULE_deprecated_metadata; } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).enterDeprecated_metadata(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof EsqlBaseParserListener ) ((EsqlBaseParserListener)listener).exitDeprecated_metadata(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof EsqlBaseParserVisitor ) return ((EsqlBaseParserVisitor)visitor).visitDeprecated_metadata(this); - else return visitor.visitChildren(this); - } - } - - public final Deprecated_metadataContext deprecated_metadata() throws RecognitionException { - Deprecated_metadataContext _localctx = new Deprecated_metadataContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_deprecated_metadata); - try { - enterOuterAlt(_localctx, 1); - { - setState(358); - match(OPENING_BRACKET); - setState(359); - metadataOption(); - setState(360); - match(CLOSING_BRACKET); - } - } - catch (RecognitionException re) { - _localctx.exception = re; - _errHandler.reportError(this, re); - _errHandler.recover(this, re); - } - finally { - exitRule(); - } - return _localctx; - } - @SuppressWarnings("CheckReturnValue") public static class MetricsCommandContext extends ParserRuleContext { public AggFieldsContext aggregates; @@ -2656,51 +2540,51 @@ public T accept(ParseTreeVisitor visitor) { public final MetricsCommandContext metricsCommand() throws RecognitionException { MetricsCommandContext _localctx = new MetricsCommandContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_metricsCommand); + enterRule(_localctx, 44, RULE_metricsCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(362); + setState(350); match(DEV_METRICS); - setState(363); + setState(351); indexPattern(); - setState(368); + setState(356); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(364); + setState(352); match(COMMA); - setState(365); + setState(353); indexPattern(); } } } - setState(370); + setState(358); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,28,_ctx); + _alt = getInterpreter().adaptivePredict(_input,27,_ctx); } - setState(372); + setState(360); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,28,_ctx) ) { case 1: { - setState(371); + setState(359); ((MetricsCommandContext)_localctx).aggregates = aggFields(); } break; } - setState(376); + setState(364); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,29,_ctx) ) { case 1: { - setState(374); + setState(362); match(BY); - setState(375); + setState(363); ((MetricsCommandContext)_localctx).grouping = fields(); } break; @@ -2746,13 +2630,13 @@ public T accept(ParseTreeVisitor visitor) { public final EvalCommandContext evalCommand() throws RecognitionException { EvalCommandContext _localctx = new EvalCommandContext(_ctx, getState()); - enterRule(_localctx, 50, RULE_evalCommand); + enterRule(_localctx, 46, RULE_evalCommand); try { enterOuterAlt(_localctx, 1); { - setState(378); + setState(366); match(EVAL); - setState(379); + setState(367); fields(); } } @@ -2801,30 +2685,30 @@ public T accept(ParseTreeVisitor visitor) { public final StatsCommandContext statsCommand() throws RecognitionException { StatsCommandContext _localctx = new StatsCommandContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_statsCommand); + enterRule(_localctx, 48, RULE_statsCommand); try { enterOuterAlt(_localctx, 1); { - setState(381); + setState(369); match(STATS); - setState(383); + setState(371); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,30,_ctx) ) { case 1: { - setState(382); + setState(370); ((StatsCommandContext)_localctx).stats = aggFields(); } break; } - setState(387); + setState(375); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,32,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,31,_ctx) ) { case 1: { - setState(385); + setState(373); match(BY); - setState(386); + setState(374); ((StatsCommandContext)_localctx).grouping = fields(); } break; @@ -2876,30 +2760,30 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldsContext aggFields() throws RecognitionException { AggFieldsContext _localctx = new AggFieldsContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_aggFields); + enterRule(_localctx, 50, RULE_aggFields); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(389); + setState(377); aggField(); - setState(394); + setState(382); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(390); + setState(378); match(COMMA); - setState(391); + setState(379); aggField(); } } } - setState(396); + setState(384); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,33,_ctx); + _alt = getInterpreter().adaptivePredict(_input,32,_ctx); } } } @@ -2945,20 +2829,20 @@ public T accept(ParseTreeVisitor visitor) { public final AggFieldContext aggField() throws RecognitionException { AggFieldContext _localctx = new AggFieldContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_aggField); + enterRule(_localctx, 52, RULE_aggField); try { enterOuterAlt(_localctx, 1); { - setState(397); + setState(385); field(); - setState(400); + setState(388); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,34,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,33,_ctx) ) { case 1: { - setState(398); + setState(386); match(WHERE); - setState(399); + setState(387); booleanExpression(0); } break; @@ -3010,30 +2894,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_qualifiedName); + enterRule(_localctx, 54, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(402); + setState(390); identifierOrParameter(); - setState(407); + setState(395); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(403); + setState(391); match(DOT); - setState(404); + setState(392); identifierOrParameter(); } } } - setState(409); + setState(397); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,35,_ctx); + _alt = getInterpreter().adaptivePredict(_input,34,_ctx); } } } @@ -3082,30 +2966,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternContext qualifiedNamePattern() throws RecognitionException { QualifiedNamePatternContext _localctx = new QualifiedNamePatternContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_qualifiedNamePattern); + enterRule(_localctx, 56, RULE_qualifiedNamePattern); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(410); + setState(398); identifierPattern(); - setState(415); + setState(403); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(411); + setState(399); match(DOT); - setState(412); + setState(400); identifierPattern(); } } } - setState(417); + setState(405); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,36,_ctx); + _alt = getInterpreter().adaptivePredict(_input,35,_ctx); } } } @@ -3154,30 +3038,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNamePatternsContext qualifiedNamePatterns() throws RecognitionException { QualifiedNamePatternsContext _localctx = new QualifiedNamePatternsContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_qualifiedNamePatterns); + enterRule(_localctx, 58, RULE_qualifiedNamePatterns); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(418); + setState(406); qualifiedNamePattern(); - setState(423); + setState(411); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(419); + setState(407); match(COMMA); - setState(420); + setState(408); qualifiedNamePattern(); } } } - setState(425); + setState(413); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,37,_ctx); + _alt = getInterpreter().adaptivePredict(_input,36,_ctx); } } } @@ -3218,12 +3102,12 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_identifier); + enterRule(_localctx, 60, RULE_identifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(426); + setState(414); _la = _input.LA(1); if ( !(_la==UNQUOTED_IDENTIFIER || _la==QUOTED_IDENTIFIER) ) { _errHandler.recoverInline(this); @@ -3274,24 +3158,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierPatternContext identifierPattern() throws RecognitionException { IdentifierPatternContext _localctx = new IdentifierPatternContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_identifierPattern); + enterRule(_localctx, 62, RULE_identifierPattern); try { - setState(431); + setState(419); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,38,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,37,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(428); + setState(416); match(ID_PATTERN); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(429); + setState(417); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(430); + setState(418); parameter(); } break; @@ -3561,17 +3445,17 @@ public T accept(ParseTreeVisitor visitor) { public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_constant); + enterRule(_localctx, 64, RULE_constant); int _la; try { - setState(475); + setState(463); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,42,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(433); + setState(421); match(NULL); } break; @@ -3579,9 +3463,9 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new QualifiedIntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(434); + setState(422); integerValue(); - setState(435); + setState(423); match(UNQUOTED_IDENTIFIER); } break; @@ -3589,7 +3473,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(437); + setState(425); decimalValue(); } break; @@ -3597,7 +3481,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(438); + setState(426); integerValue(); } break; @@ -3605,7 +3489,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(439); + setState(427); booleanValue(); } break; @@ -3613,7 +3497,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new InputParameterContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(440); + setState(428); parameter(); } break; @@ -3621,7 +3505,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(441); + setState(429); string(); } break; @@ -3629,27 +3513,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(442); + setState(430); match(OPENING_BRACKET); - setState(443); + setState(431); numericValue(); - setState(448); + setState(436); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(444); + setState(432); match(COMMA); - setState(445); + setState(433); numericValue(); } } - setState(450); + setState(438); _errHandler.sync(this); _la = _input.LA(1); } - setState(451); + setState(439); match(CLOSING_BRACKET); } break; @@ -3657,27 +3541,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(453); + setState(441); match(OPENING_BRACKET); - setState(454); + setState(442); booleanValue(); - setState(459); + setState(447); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(455); + setState(443); match(COMMA); - setState(456); + setState(444); booleanValue(); } } - setState(461); + setState(449); _errHandler.sync(this); _la = _input.LA(1); } - setState(462); + setState(450); match(CLOSING_BRACKET); } break; @@ -3685,27 +3569,27 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringArrayLiteralContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(464); + setState(452); match(OPENING_BRACKET); - setState(465); + setState(453); string(); - setState(470); + setState(458); _errHandler.sync(this); _la = _input.LA(1); while (_la==COMMA) { { { - setState(466); + setState(454); match(COMMA); - setState(467); + setState(455); string(); } } - setState(472); + setState(460); _errHandler.sync(this); _la = _input.LA(1); } - setState(473); + setState(461); match(CLOSING_BRACKET); } break; @@ -3777,16 +3661,16 @@ public T accept(ParseTreeVisitor visitor) { public final ParameterContext parameter() throws RecognitionException { ParameterContext _localctx = new ParameterContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_parameter); + enterRule(_localctx, 66, RULE_parameter); try { - setState(479); + setState(467); _errHandler.sync(this); switch (_input.LA(1)) { case PARAM: _localctx = new InputParamContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(477); + setState(465); match(PARAM); } break; @@ -3794,7 +3678,7 @@ public final ParameterContext parameter() throws RecognitionException { _localctx = new InputNamedOrPositionalParamContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(478); + setState(466); match(NAMED_OR_POSITIONAL_PARAM); } break; @@ -3843,24 +3727,24 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierOrParameterContext identifierOrParameter() throws RecognitionException { IdentifierOrParameterContext _localctx = new IdentifierOrParameterContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_identifierOrParameter); + enterRule(_localctx, 68, RULE_identifierOrParameter); try { - setState(484); + setState(472); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,44,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,43,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(481); + setState(469); identifier(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(482); + setState(470); if (!(this.isDevVersion())) throw new FailedPredicateException(this, "this.isDevVersion()"); - setState(483); + setState(471); parameter(); } break; @@ -3903,13 +3787,13 @@ public T accept(ParseTreeVisitor visitor) { public final LimitCommandContext limitCommand() throws RecognitionException { LimitCommandContext _localctx = new LimitCommandContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_limitCommand); + enterRule(_localctx, 70, RULE_limitCommand); try { enterOuterAlt(_localctx, 1); { - setState(486); + setState(474); match(LIMIT); - setState(487); + setState(475); match(INTEGER_LITERAL); } } @@ -3959,32 +3843,32 @@ public T accept(ParseTreeVisitor visitor) { public final SortCommandContext sortCommand() throws RecognitionException { SortCommandContext _localctx = new SortCommandContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_sortCommand); + enterRule(_localctx, 72, RULE_sortCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(489); + setState(477); match(SORT); - setState(490); + setState(478); orderExpression(); - setState(495); + setState(483); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(491); + setState(479); match(COMMA); - setState(492); + setState(480); orderExpression(); } } } - setState(497); + setState(485); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,45,_ctx); + _alt = getInterpreter().adaptivePredict(_input,44,_ctx); } } } @@ -4033,19 +3917,19 @@ public T accept(ParseTreeVisitor visitor) { public final OrderExpressionContext orderExpression() throws RecognitionException { OrderExpressionContext _localctx = new OrderExpressionContext(_ctx, getState()); - enterRule(_localctx, 78, RULE_orderExpression); + enterRule(_localctx, 74, RULE_orderExpression); int _la; try { enterOuterAlt(_localctx, 1); { - setState(498); + setState(486); booleanExpression(0); - setState(500); + setState(488); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,45,_ctx) ) { case 1: { - setState(499); + setState(487); ((OrderExpressionContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -4059,14 +3943,14 @@ public final OrderExpressionContext orderExpression() throws RecognitionExceptio } break; } - setState(504); + setState(492); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,47,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,46,_ctx) ) { case 1: { - setState(502); + setState(490); match(NULLS); - setState(503); + setState(491); ((OrderExpressionContext)_localctx).nullOrdering = _input.LT(1); _la = _input.LA(1); if ( !(_la==FIRST || _la==LAST) ) { @@ -4121,13 +4005,13 @@ public T accept(ParseTreeVisitor visitor) { public final KeepCommandContext keepCommand() throws RecognitionException { KeepCommandContext _localctx = new KeepCommandContext(_ctx, getState()); - enterRule(_localctx, 80, RULE_keepCommand); + enterRule(_localctx, 76, RULE_keepCommand); try { enterOuterAlt(_localctx, 1); { - setState(506); + setState(494); match(KEEP); - setState(507); + setState(495); qualifiedNamePatterns(); } } @@ -4170,13 +4054,13 @@ public T accept(ParseTreeVisitor visitor) { public final DropCommandContext dropCommand() throws RecognitionException { DropCommandContext _localctx = new DropCommandContext(_ctx, getState()); - enterRule(_localctx, 82, RULE_dropCommand); + enterRule(_localctx, 78, RULE_dropCommand); try { enterOuterAlt(_localctx, 1); { - setState(509); + setState(497); match(DROP); - setState(510); + setState(498); qualifiedNamePatterns(); } } @@ -4226,32 +4110,32 @@ public T accept(ParseTreeVisitor visitor) { public final RenameCommandContext renameCommand() throws RecognitionException { RenameCommandContext _localctx = new RenameCommandContext(_ctx, getState()); - enterRule(_localctx, 84, RULE_renameCommand); + enterRule(_localctx, 80, RULE_renameCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(512); + setState(500); match(RENAME); - setState(513); + setState(501); renameClause(); - setState(518); + setState(506); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(514); + setState(502); match(COMMA); - setState(515); + setState(503); renameClause(); } } } - setState(520); + setState(508); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,48,_ctx); + _alt = getInterpreter().adaptivePredict(_input,47,_ctx); } } } @@ -4299,15 +4183,15 @@ public T accept(ParseTreeVisitor visitor) { public final RenameClauseContext renameClause() throws RecognitionException { RenameClauseContext _localctx = new RenameClauseContext(_ctx, getState()); - enterRule(_localctx, 86, RULE_renameClause); + enterRule(_localctx, 82, RULE_renameClause); try { enterOuterAlt(_localctx, 1); { - setState(521); + setState(509); ((RenameClauseContext)_localctx).oldName = qualifiedNamePattern(); - setState(522); + setState(510); match(AS); - setState(523); + setState(511); ((RenameClauseContext)_localctx).newName = qualifiedNamePattern(); } } @@ -4356,22 +4240,22 @@ public T accept(ParseTreeVisitor visitor) { public final DissectCommandContext dissectCommand() throws RecognitionException { DissectCommandContext _localctx = new DissectCommandContext(_ctx, getState()); - enterRule(_localctx, 88, RULE_dissectCommand); + enterRule(_localctx, 84, RULE_dissectCommand); try { enterOuterAlt(_localctx, 1); { - setState(525); + setState(513); match(DISSECT); - setState(526); + setState(514); primaryExpression(0); - setState(527); + setState(515); string(); - setState(529); + setState(517); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,49,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,48,_ctx) ) { case 1: { - setState(528); + setState(516); commandOptions(); } break; @@ -4420,15 +4304,15 @@ public T accept(ParseTreeVisitor visitor) { public final GrokCommandContext grokCommand() throws RecognitionException { GrokCommandContext _localctx = new GrokCommandContext(_ctx, getState()); - enterRule(_localctx, 90, RULE_grokCommand); + enterRule(_localctx, 86, RULE_grokCommand); try { enterOuterAlt(_localctx, 1); { - setState(531); + setState(519); match(GROK); - setState(532); + setState(520); primaryExpression(0); - setState(533); + setState(521); string(); } } @@ -4471,13 +4355,13 @@ public T accept(ParseTreeVisitor visitor) { public final MvExpandCommandContext mvExpandCommand() throws RecognitionException { MvExpandCommandContext _localctx = new MvExpandCommandContext(_ctx, getState()); - enterRule(_localctx, 92, RULE_mvExpandCommand); + enterRule(_localctx, 88, RULE_mvExpandCommand); try { enterOuterAlt(_localctx, 1); { - setState(535); + setState(523); match(MV_EXPAND); - setState(536); + setState(524); qualifiedName(); } } @@ -4526,30 +4410,30 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionsContext commandOptions() throws RecognitionException { CommandOptionsContext _localctx = new CommandOptionsContext(_ctx, getState()); - enterRule(_localctx, 94, RULE_commandOptions); + enterRule(_localctx, 90, RULE_commandOptions); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(538); + setState(526); commandOption(); - setState(543); + setState(531); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(539); + setState(527); match(COMMA); - setState(540); + setState(528); commandOption(); } } } - setState(545); + setState(533); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,50,_ctx); + _alt = getInterpreter().adaptivePredict(_input,49,_ctx); } } } @@ -4595,15 +4479,15 @@ public T accept(ParseTreeVisitor visitor) { public final CommandOptionContext commandOption() throws RecognitionException { CommandOptionContext _localctx = new CommandOptionContext(_ctx, getState()); - enterRule(_localctx, 96, RULE_commandOption); + enterRule(_localctx, 92, RULE_commandOption); try { enterOuterAlt(_localctx, 1); { - setState(546); + setState(534); identifier(); - setState(547); + setState(535); match(ASSIGN); - setState(548); + setState(536); constant(); } } @@ -4644,12 +4528,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 98, RULE_booleanValue); + enterRule(_localctx, 94, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(550); + setState(538); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4702,22 +4586,22 @@ public T accept(ParseTreeVisitor visitor) { public final NumericValueContext numericValue() throws RecognitionException { NumericValueContext _localctx = new NumericValueContext(_ctx, getState()); - enterRule(_localctx, 100, RULE_numericValue); + enterRule(_localctx, 96, RULE_numericValue); try { - setState(554); + setState(542); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,51,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,50,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(552); + setState(540); decimalValue(); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(553); + setState(541); integerValue(); } break; @@ -4761,17 +4645,17 @@ public T accept(ParseTreeVisitor visitor) { public final DecimalValueContext decimalValue() throws RecognitionException { DecimalValueContext _localctx = new DecimalValueContext(_ctx, getState()); - enterRule(_localctx, 102, RULE_decimalValue); + enterRule(_localctx, 98, RULE_decimalValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(557); + setState(545); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(556); + setState(544); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4784,7 +4668,7 @@ public final DecimalValueContext decimalValue() throws RecognitionException { } } - setState(559); + setState(547); match(DECIMAL_LITERAL); } } @@ -4826,17 +4710,17 @@ public T accept(ParseTreeVisitor visitor) { public final IntegerValueContext integerValue() throws RecognitionException { IntegerValueContext _localctx = new IntegerValueContext(_ctx, getState()); - enterRule(_localctx, 104, RULE_integerValue); + enterRule(_localctx, 100, RULE_integerValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(562); + setState(550); _errHandler.sync(this); _la = _input.LA(1); if (_la==PLUS || _la==MINUS) { { - setState(561); + setState(549); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { _errHandler.recoverInline(this); @@ -4849,7 +4733,7 @@ public final IntegerValueContext integerValue() throws RecognitionException { } } - setState(564); + setState(552); match(INTEGER_LITERAL); } } @@ -4889,11 +4773,11 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 106, RULE_string); + enterRule(_localctx, 102, RULE_string); try { enterOuterAlt(_localctx, 1); { - setState(566); + setState(554); match(QUOTED_STRING); } } @@ -4938,12 +4822,12 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 108, RULE_comparisonOperator); + enterRule(_localctx, 104, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(568); + setState(556); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & -432345564227567616L) != 0)) ) { _errHandler.recoverInline(this); @@ -4994,13 +4878,13 @@ public T accept(ParseTreeVisitor visitor) { public final ExplainCommandContext explainCommand() throws RecognitionException { ExplainCommandContext _localctx = new ExplainCommandContext(_ctx, getState()); - enterRule(_localctx, 110, RULE_explainCommand); + enterRule(_localctx, 106, RULE_explainCommand); try { enterOuterAlt(_localctx, 1); { - setState(570); + setState(558); match(EXPLAIN); - setState(571); + setState(559); subqueryExpression(); } } @@ -5044,15 +4928,15 @@ public T accept(ParseTreeVisitor visitor) { public final SubqueryExpressionContext subqueryExpression() throws RecognitionException { SubqueryExpressionContext _localctx = new SubqueryExpressionContext(_ctx, getState()); - enterRule(_localctx, 112, RULE_subqueryExpression); + enterRule(_localctx, 108, RULE_subqueryExpression); try { enterOuterAlt(_localctx, 1); { - setState(573); + setState(561); match(OPENING_BRACKET); - setState(574); + setState(562); query(0); - setState(575); + setState(563); match(CLOSING_BRACKET); } } @@ -5104,14 +4988,14 @@ public T accept(ParseTreeVisitor visitor) { public final ShowCommandContext showCommand() throws RecognitionException { ShowCommandContext _localctx = new ShowCommandContext(_ctx, getState()); - enterRule(_localctx, 114, RULE_showCommand); + enterRule(_localctx, 110, RULE_showCommand); try { _localctx = new ShowInfoContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(577); + setState(565); match(SHOW); - setState(578); + setState(566); match(INFO); } } @@ -5169,53 +5053,53 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichCommandContext enrichCommand() throws RecognitionException { EnrichCommandContext _localctx = new EnrichCommandContext(_ctx, getState()); - enterRule(_localctx, 116, RULE_enrichCommand); + enterRule(_localctx, 112, RULE_enrichCommand); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(580); + setState(568); match(ENRICH); - setState(581); + setState(569); ((EnrichCommandContext)_localctx).policyName = match(ENRICH_POLICY_NAME); - setState(584); + setState(572); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,54,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,53,_ctx) ) { case 1: { - setState(582); + setState(570); match(ON); - setState(583); + setState(571); ((EnrichCommandContext)_localctx).matchField = qualifiedNamePattern(); } break; } - setState(595); + setState(583); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,55,_ctx) ) { case 1: { - setState(586); + setState(574); match(WITH); - setState(587); + setState(575); enrichWithClause(); - setState(592); + setState(580); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(588); + setState(576); match(COMMA); - setState(589); + setState(577); enrichWithClause(); } } } - setState(594); + setState(582); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,55,_ctx); + _alt = getInterpreter().adaptivePredict(_input,54,_ctx); } } break; @@ -5266,23 +5150,23 @@ public T accept(ParseTreeVisitor visitor) { public final EnrichWithClauseContext enrichWithClause() throws RecognitionException { EnrichWithClauseContext _localctx = new EnrichWithClauseContext(_ctx, getState()); - enterRule(_localctx, 118, RULE_enrichWithClause); + enterRule(_localctx, 114, RULE_enrichWithClause); try { enterOuterAlt(_localctx, 1); { - setState(600); + setState(588); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,56,_ctx) ) { case 1: { - setState(597); + setState(585); ((EnrichWithClauseContext)_localctx).newName = qualifiedNamePattern(); - setState(598); + setState(586); match(ASSIGN); } break; } - setState(602); + setState(590); ((EnrichWithClauseContext)_localctx).enrichField = qualifiedNamePattern(); } } @@ -5331,17 +5215,17 @@ public T accept(ParseTreeVisitor visitor) { public final LookupCommandContext lookupCommand() throws RecognitionException { LookupCommandContext _localctx = new LookupCommandContext(_ctx, getState()); - enterRule(_localctx, 120, RULE_lookupCommand); + enterRule(_localctx, 116, RULE_lookupCommand); try { enterOuterAlt(_localctx, 1); { - setState(604); + setState(592); match(DEV_LOOKUP); - setState(605); + setState(593); ((LookupCommandContext)_localctx).tableName = indexPattern(); - setState(606); + setState(594); match(ON); - setState(607); + setState(595); ((LookupCommandContext)_localctx).matchFields = qualifiedNamePatterns(); } } @@ -5390,22 +5274,22 @@ public T accept(ParseTreeVisitor visitor) { public final InlinestatsCommandContext inlinestatsCommand() throws RecognitionException { InlinestatsCommandContext _localctx = new InlinestatsCommandContext(_ctx, getState()); - enterRule(_localctx, 122, RULE_inlinestatsCommand); + enterRule(_localctx, 118, RULE_inlinestatsCommand); try { enterOuterAlt(_localctx, 1); { - setState(609); + setState(597); match(DEV_INLINESTATS); - setState(610); + setState(598); ((InlinestatsCommandContext)_localctx).stats = aggFields(); - setState(613); + setState(601); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,58,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,57,_ctx) ) { case 1: { - setState(611); + setState(599); match(BY); - setState(612); + setState(600); ((InlinestatsCommandContext)_localctx).grouping = fields(); } break; @@ -5458,17 +5342,17 @@ public T accept(ParseTreeVisitor visitor) { public final JoinCommandContext joinCommand() throws RecognitionException { JoinCommandContext _localctx = new JoinCommandContext(_ctx, getState()); - enterRule(_localctx, 124, RULE_joinCommand); + enterRule(_localctx, 120, RULE_joinCommand); int _la; try { enterOuterAlt(_localctx, 1); { - setState(616); + setState(604); _errHandler.sync(this); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) { { - setState(615); + setState(603); ((JoinCommandContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & 29360128L) != 0)) ) { @@ -5482,11 +5366,11 @@ public final JoinCommandContext joinCommand() throws RecognitionException { } } - setState(618); + setState(606); match(DEV_JOIN); - setState(619); + setState(607); joinTarget(); - setState(620); + setState(608); joinCondition(); } } @@ -5534,21 +5418,21 @@ public T accept(ParseTreeVisitor visitor) { public final JoinTargetContext joinTarget() throws RecognitionException { JoinTargetContext _localctx = new JoinTargetContext(_ctx, getState()); - enterRule(_localctx, 126, RULE_joinTarget); + enterRule(_localctx, 122, RULE_joinTarget); int _la; try { enterOuterAlt(_localctx, 1); { - setState(622); + setState(610); ((JoinTargetContext)_localctx).index = identifier(); - setState(625); + setState(613); _errHandler.sync(this); _la = _input.LA(1); if (_la==AS) { { - setState(623); + setState(611); match(AS); - setState(624); + setState(612); ((JoinTargetContext)_localctx).alias = identifier(); } } @@ -5601,32 +5485,32 @@ public T accept(ParseTreeVisitor visitor) { public final JoinConditionContext joinCondition() throws RecognitionException { JoinConditionContext _localctx = new JoinConditionContext(_ctx, getState()); - enterRule(_localctx, 128, RULE_joinCondition); + enterRule(_localctx, 124, RULE_joinCondition); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(627); + setState(615); match(ON); - setState(628); + setState(616); joinPredicate(); - setState(633); + setState(621); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(629); + setState(617); match(COMMA); - setState(630); + setState(618); joinPredicate(); } } } - setState(635); + setState(623); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,61,_ctx); + _alt = getInterpreter().adaptivePredict(_input,60,_ctx); } } } @@ -5668,11 +5552,11 @@ public T accept(ParseTreeVisitor visitor) { public final JoinPredicateContext joinPredicate() throws RecognitionException { JoinPredicateContext _localctx = new JoinPredicateContext(_ctx, getState()); - enterRule(_localctx, 130, RULE_joinPredicate); + enterRule(_localctx, 126, RULE_joinPredicate); try { enterOuterAlt(_localctx, 1); { - setState(636); + setState(624); valueExpression(); } } @@ -5701,9 +5585,9 @@ public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { return operatorExpression_sempred((OperatorExpressionContext)_localctx, predIndex); case 10: return primaryExpression_sempred((PrimaryExpressionContext)_localctx, predIndex); - case 33: + case 31: return identifierPattern_sempred((IdentifierPatternContext)_localctx, predIndex); - case 36: + case 34: return identifierOrParameter_sempred((IdentifierOrParameterContext)_localctx, predIndex); } return true; @@ -5774,7 +5658,7 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca } public static final String _serializedATN = - "\u0004\u0001\u0080\u027f\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ + "\u0004\u0001\u0080\u0273\u0002\u0000\u0007\u0000\u0002\u0001\u0007\u0001"+ "\u0002\u0002\u0007\u0002\u0002\u0003\u0007\u0003\u0002\u0004\u0007\u0004"+ "\u0002\u0005\u0007\u0005\u0002\u0006\u0007\u0006\u0002\u0007\u0007\u0007"+ "\u0002\b\u0007\b\u0002\t\u0007\t\u0002\n\u0007\n\u0002\u000b\u0007\u000b"+ @@ -5790,392 +5674,385 @@ private boolean identifierOrParameter_sempred(IdentifierOrParameterContext _loca ",\u0002-\u0007-\u0002.\u0007.\u0002/\u0007/\u00020\u00070\u00021\u0007"+ "1\u00022\u00072\u00023\u00073\u00024\u00074\u00025\u00075\u00026\u0007"+ "6\u00027\u00077\u00028\u00078\u00029\u00079\u0002:\u0007:\u0002;\u0007"+ - ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0002@\u0007"+ - "@\u0002A\u0007A\u0001\u0000\u0001\u0000\u0001\u0000\u0001\u0001\u0001"+ - "\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0005\u0001\u008e"+ - "\b\u0001\n\u0001\f\u0001\u0091\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002"+ - "\u0001\u0002\u0001\u0002\u0001\u0002\u0003\u0002\u0099\b\u0002\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003"+ - "\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0003\u0003"+ - "\u00ad\b\u0003\u0001\u0004\u0001\u0004\u0001\u0004\u0001\u0005\u0001\u0005"+ - "\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005"+ - "\u00b9\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005"+ - "\u0005\u0005\u00c0\b\u0005\n\u0005\f\u0005\u00c3\t\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00ca\b\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00cf\b\u0005\u0001\u0005\u0001"+ - "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d7"+ - "\b\u0005\n\u0005\f\u0005\u00da\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00de\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006"+ - "\u0003\u0006\u00e5\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006"+ - "\u00ea\b\u0006\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00ef\b"+ - "\u0007\u0001\u0007\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001"+ - "\b\u0001\b\u0003\b\u00f9\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00ff"+ - "\b\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0107\b\t"+ - "\n\t\f\t\u010a\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001"+ - "\n\u0001\n\u0003\n\u0114\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0119\b\n"+ - "\n\n\f\n\u011c\t\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001"+ - "\u000b\u0001\u000b\u0005\u000b\u0124\b\u000b\n\u000b\f\u000b\u0127\t\u000b"+ - "\u0003\u000b\u0129\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001"+ - "\r\u0001\r\u0001\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f"+ - "\u0001\u000f\u0005\u000f\u0137\b\u000f\n\u000f\f\u000f\u013a\t\u000f\u0001"+ - "\u0010\u0001\u0010\u0001\u0010\u0003\u0010\u013f\b\u0010\u0001\u0010\u0001"+ - "\u0010\u0001\u0011\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0147"+ - "\b\u0011\n\u0011\f\u0011\u014a\t\u0011\u0001\u0011\u0003\u0011\u014d\b"+ - "\u0011\u0001\u0012\u0001\u0012\u0001\u0012\u0003\u0012\u0152\b\u0012\u0001"+ - "\u0012\u0001\u0012\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001"+ - "\u0015\u0001\u0015\u0003\u0015\u015c\b\u0015\u0001\u0016\u0001\u0016\u0001"+ - "\u0016\u0001\u0016\u0005\u0016\u0162\b\u0016\n\u0016\f\u0016\u0165\t\u0016"+ - "\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0017\u0001\u0018\u0001\u0018"+ - "\u0001\u0018\u0001\u0018\u0005\u0018\u016f\b\u0018\n\u0018\f\u0018\u0172"+ - "\t\u0018\u0001\u0018\u0003\u0018\u0175\b\u0018\u0001\u0018\u0001\u0018"+ - "\u0003\u0018\u0179\b\u0018\u0001\u0019\u0001\u0019\u0001\u0019\u0001\u001a"+ - "\u0001\u001a\u0003\u001a\u0180\b\u001a\u0001\u001a\u0001\u001a\u0003\u001a"+ - "\u0184\b\u001a\u0001\u001b\u0001\u001b\u0001\u001b\u0005\u001b\u0189\b"+ - "\u001b\n\u001b\f\u001b\u018c\t\u001b\u0001\u001c\u0001\u001c\u0001\u001c"+ - "\u0003\u001c\u0191\b\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d"+ - "\u0196\b\u001d\n\u001d\f\u001d\u0199\t\u001d\u0001\u001e\u0001\u001e\u0001"+ - "\u001e\u0005\u001e\u019e\b\u001e\n\u001e\f\u001e\u01a1\t\u001e\u0001\u001f"+ - "\u0001\u001f\u0001\u001f\u0005\u001f\u01a6\b\u001f\n\u001f\f\u001f\u01a9"+ - "\t\u001f\u0001 \u0001 \u0001!\u0001!\u0001!\u0003!\u01b0\b!\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001"+ - "\"\u0001\"\u0001\"\u0005\"\u01bf\b\"\n\"\f\"\u01c2\t\"\u0001\"\u0001\""+ - "\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01ca\b\"\n\"\f\"\u01cd\t\"\u0001"+ - "\"\u0001\"\u0001\"\u0001\"\u0001\"\u0001\"\u0005\"\u01d5\b\"\n\"\f\"\u01d8"+ - "\t\"\u0001\"\u0001\"\u0003\"\u01dc\b\"\u0001#\u0001#\u0003#\u01e0\b#\u0001"+ - "$\u0001$\u0001$\u0003$\u01e5\b$\u0001%\u0001%\u0001%\u0001&\u0001&\u0001"+ - "&\u0001&\u0005&\u01ee\b&\n&\f&\u01f1\t&\u0001\'\u0001\'\u0003\'\u01f5"+ - "\b\'\u0001\'\u0001\'\u0003\'\u01f9\b\'\u0001(\u0001(\u0001(\u0001)\u0001"+ - ")\u0001)\u0001*\u0001*\u0001*\u0001*\u0005*\u0205\b*\n*\f*\u0208\t*\u0001"+ - "+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001,\u0001,\u0003,\u0212\b,\u0001"+ - "-\u0001-\u0001-\u0001-\u0001.\u0001.\u0001.\u0001/\u0001/\u0001/\u0005"+ - "/\u021e\b/\n/\f/\u0221\t/\u00010\u00010\u00010\u00010\u00011\u00011\u0001"+ - "2\u00012\u00032\u022b\b2\u00013\u00033\u022e\b3\u00013\u00013\u00014\u0003"+ - "4\u0233\b4\u00014\u00014\u00015\u00015\u00016\u00016\u00017\u00017\u0001"+ - "7\u00018\u00018\u00018\u00018\u00019\u00019\u00019\u0001:\u0001:\u0001"+ - ":\u0001:\u0003:\u0249\b:\u0001:\u0001:\u0001:\u0001:\u0005:\u024f\b:\n"+ - ":\f:\u0252\t:\u0003:\u0254\b:\u0001;\u0001;\u0001;\u0003;\u0259\b;\u0001"+ - ";\u0001;\u0001<\u0001<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0001"+ - "=\u0003=\u0266\b=\u0001>\u0003>\u0269\b>\u0001>\u0001>\u0001>\u0001>\u0001"+ - "?\u0001?\u0001?\u0003?\u0272\b?\u0001@\u0001@\u0001@\u0001@\u0005@\u0278"+ - "\b@\n@\f@\u027b\t@\u0001A\u0001A\u0001A\u0000\u0004\u0002\n\u0012\u0014"+ - "B\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010\u0012\u0014\u0016\u0018\u001a"+ - "\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPRTVXZ\\^`bdfhjlnprtvxz|~\u0080\u0082"+ - "\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000\u001e\u001eQQ\u0001\u0000"+ - "HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000**88\u0002\u000099;?\u0001"+ - "\u0000\u0016\u0018\u0299\u0000\u0084\u0001\u0000\u0000\u0000\u0002\u0087"+ - "\u0001\u0000\u0000\u0000\u0004\u0098\u0001\u0000\u0000\u0000\u0006\u00ac"+ - "\u0001\u0000\u0000\u0000\b\u00ae\u0001\u0000\u0000\u0000\n\u00ce\u0001"+ - "\u0000\u0000\u0000\f\u00e9\u0001\u0000\u0000\u0000\u000e\u00eb\u0001\u0000"+ - "\u0000\u0000\u0010\u00f8\u0001\u0000\u0000\u0000\u0012\u00fe\u0001\u0000"+ - "\u0000\u0000\u0014\u0113\u0001\u0000\u0000\u0000\u0016\u011d\u0001\u0000"+ - "\u0000\u0000\u0018\u012c\u0001\u0000\u0000\u0000\u001a\u012e\u0001\u0000"+ - "\u0000\u0000\u001c\u0130\u0001\u0000\u0000\u0000\u001e\u0133\u0001\u0000"+ - "\u0000\u0000 \u013e\u0001\u0000\u0000\u0000\"\u0142\u0001\u0000\u0000"+ - "\u0000$\u0151\u0001\u0000\u0000\u0000&\u0155\u0001\u0000\u0000\u0000("+ - "\u0157\u0001\u0000\u0000\u0000*\u015b\u0001\u0000\u0000\u0000,\u015d\u0001"+ - "\u0000\u0000\u0000.\u0166\u0001\u0000\u0000\u00000\u016a\u0001\u0000\u0000"+ - "\u00002\u017a\u0001\u0000\u0000\u00004\u017d\u0001\u0000\u0000\u00006"+ - "\u0185\u0001\u0000\u0000\u00008\u018d\u0001\u0000\u0000\u0000:\u0192\u0001"+ - "\u0000\u0000\u0000<\u019a\u0001\u0000\u0000\u0000>\u01a2\u0001\u0000\u0000"+ - "\u0000@\u01aa\u0001\u0000\u0000\u0000B\u01af\u0001\u0000\u0000\u0000D"+ - "\u01db\u0001\u0000\u0000\u0000F\u01df\u0001\u0000\u0000\u0000H\u01e4\u0001"+ - "\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L\u01e9\u0001\u0000\u0000"+ - "\u0000N\u01f2\u0001\u0000\u0000\u0000P\u01fa\u0001\u0000\u0000\u0000R"+ - "\u01fd\u0001\u0000\u0000\u0000T\u0200\u0001\u0000\u0000\u0000V\u0209\u0001"+ - "\u0000\u0000\u0000X\u020d\u0001\u0000\u0000\u0000Z\u0213\u0001\u0000\u0000"+ - "\u0000\\\u0217\u0001\u0000\u0000\u0000^\u021a\u0001\u0000\u0000\u0000"+ - "`\u0222\u0001\u0000\u0000\u0000b\u0226\u0001\u0000\u0000\u0000d\u022a"+ - "\u0001\u0000\u0000\u0000f\u022d\u0001\u0000\u0000\u0000h\u0232\u0001\u0000"+ - "\u0000\u0000j\u0236\u0001\u0000\u0000\u0000l\u0238\u0001\u0000\u0000\u0000"+ - "n\u023a\u0001\u0000\u0000\u0000p\u023d\u0001\u0000\u0000\u0000r\u0241"+ - "\u0001\u0000\u0000\u0000t\u0244\u0001\u0000\u0000\u0000v\u0258\u0001\u0000"+ - "\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0261\u0001\u0000\u0000\u0000"+ - "|\u0268\u0001\u0000\u0000\u0000~\u026e\u0001\u0000\u0000\u0000\u0080\u0273"+ - "\u0001\u0000\u0000\u0000\u0082\u027c\u0001\u0000\u0000\u0000\u0084\u0085"+ - "\u0003\u0002\u0001\u0000\u0085\u0086\u0005\u0000\u0000\u0001\u0086\u0001"+ - "\u0001\u0000\u0000\u0000\u0087\u0088\u0006\u0001\uffff\uffff\u0000\u0088"+ - "\u0089\u0003\u0004\u0002\u0000\u0089\u008f\u0001\u0000\u0000\u0000\u008a"+ - "\u008b\n\u0001\u0000\u0000\u008b\u008c\u0005\u001d\u0000\u0000\u008c\u008e"+ - "\u0003\u0006\u0003\u0000\u008d\u008a\u0001\u0000\u0000\u0000\u008e\u0091"+ - "\u0001\u0000\u0000\u0000\u008f\u008d\u0001\u0000\u0000\u0000\u008f\u0090"+ - "\u0001\u0000\u0000\u0000\u0090\u0003\u0001\u0000\u0000\u0000\u0091\u008f"+ - "\u0001\u0000\u0000\u0000\u0092\u0099\u0003n7\u0000\u0093\u0099\u0003\""+ - "\u0011\u0000\u0094\u0099\u0003\u001c\u000e\u0000\u0095\u0099\u0003r9\u0000"+ - "\u0096\u0097\u0004\u0002\u0001\u0000\u0097\u0099\u00030\u0018\u0000\u0098"+ - "\u0092\u0001\u0000\u0000\u0000\u0098\u0093\u0001\u0000\u0000\u0000\u0098"+ - "\u0094\u0001\u0000\u0000\u0000\u0098\u0095\u0001\u0000\u0000\u0000\u0098"+ - "\u0096\u0001\u0000\u0000\u0000\u0099\u0005\u0001\u0000\u0000\u0000\u009a"+ - "\u00ad\u00032\u0019\u0000\u009b\u00ad\u0003\b\u0004\u0000\u009c\u00ad"+ - "\u0003P(\u0000\u009d\u00ad\u0003J%\u0000\u009e\u00ad\u00034\u001a\u0000"+ - "\u009f\u00ad\u0003L&\u0000\u00a0\u00ad\u0003R)\u0000\u00a1\u00ad\u0003"+ - "T*\u0000\u00a2\u00ad\u0003X,\u0000\u00a3\u00ad\u0003Z-\u0000\u00a4\u00ad"+ - "\u0003t:\u0000\u00a5\u00ad\u0003\\.\u0000\u00a6\u00a7\u0004\u0003\u0002"+ - "\u0000\u00a7\u00ad\u0003z=\u0000\u00a8\u00a9\u0004\u0003\u0003\u0000\u00a9"+ - "\u00ad\u0003x<\u0000\u00aa\u00ab\u0004\u0003\u0004\u0000\u00ab\u00ad\u0003"+ - "|>\u0000\u00ac\u009a\u0001\u0000\u0000\u0000\u00ac\u009b\u0001\u0000\u0000"+ - "\u0000\u00ac\u009c\u0001\u0000\u0000\u0000\u00ac\u009d\u0001\u0000\u0000"+ - "\u0000\u00ac\u009e\u0001\u0000\u0000\u0000\u00ac\u009f\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a0\u0001\u0000\u0000\u0000\u00ac\u00a1\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a2\u0001\u0000\u0000\u0000\u00ac\u00a3\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a4\u0001\u0000\u0000\u0000\u00ac\u00a5\u0001\u0000\u0000"+ - "\u0000\u00ac\u00a6\u0001\u0000\u0000\u0000\u00ac\u00a8\u0001\u0000\u0000"+ - "\u0000\u00ac\u00aa\u0001\u0000\u0000\u0000\u00ad\u0007\u0001\u0000\u0000"+ - "\u0000\u00ae\u00af\u0005\u0010\u0000\u0000\u00af\u00b0\u0003\n\u0005\u0000"+ - "\u00b0\t\u0001\u0000\u0000\u0000\u00b1\u00b2\u0006\u0005\uffff\uffff\u0000"+ - "\u00b2\u00b3\u00051\u0000\u0000\u00b3\u00cf\u0003\n\u0005\b\u00b4\u00cf"+ - "\u0003\u0010\b\u0000\u00b5\u00cf\u0003\f\u0006\u0000\u00b6\u00b8\u0003"+ - "\u0010\b\u0000\u00b7\u00b9\u00051\u0000\u0000\u00b8\u00b7\u0001\u0000"+ - "\u0000\u0000\u00b8\u00b9\u0001\u0000\u0000\u0000\u00b9\u00ba\u0001\u0000"+ - "\u0000\u0000\u00ba\u00bb\u0005,\u0000\u0000\u00bb\u00bc\u00050\u0000\u0000"+ - "\u00bc\u00c1\u0003\u0010\b\u0000\u00bd\u00be\u0005\'\u0000\u0000\u00be"+ - "\u00c0\u0003\u0010\b\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c3"+ - "\u0001\u0000\u0000\u0000\u00c1\u00bf\u0001\u0000\u0000\u0000\u00c1\u00c2"+ - "\u0001\u0000\u0000\u0000\u00c2\u00c4\u0001\u0000\u0000\u0000\u00c3\u00c1"+ - "\u0001\u0000\u0000\u0000\u00c4\u00c5\u00057\u0000\u0000\u00c5\u00cf\u0001"+ - "\u0000\u0000\u0000\u00c6\u00c7\u0003\u0010\b\u0000\u00c7\u00c9\u0005-"+ - "\u0000\u0000\u00c8\u00ca\u00051\u0000\u0000\u00c9\u00c8\u0001\u0000\u0000"+ - "\u0000\u00c9\u00ca\u0001\u0000\u0000\u0000\u00ca\u00cb\u0001\u0000\u0000"+ - "\u0000\u00cb\u00cc\u00052\u0000\u0000\u00cc\u00cf\u0001\u0000\u0000\u0000"+ - "\u00cd\u00cf\u0003\u000e\u0007\u0000\u00ce\u00b1\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b4\u0001\u0000\u0000\u0000\u00ce\u00b5\u0001\u0000\u0000\u0000"+ - "\u00ce\u00b6\u0001\u0000\u0000\u0000\u00ce\u00c6\u0001\u0000\u0000\u0000"+ - "\u00ce\u00cd\u0001\u0000\u0000\u0000\u00cf\u00d8\u0001\u0000\u0000\u0000"+ - "\u00d0\u00d1\n\u0005\u0000\u0000\u00d1\u00d2\u0005\"\u0000\u0000\u00d2"+ - "\u00d7\u0003\n\u0005\u0006\u00d3\u00d4\n\u0004\u0000\u0000\u00d4\u00d5"+ - "\u00054\u0000\u0000\u00d5\u00d7\u0003\n\u0005\u0005\u00d6\u00d0\u0001"+ - "\u0000\u0000\u0000\u00d6\u00d3\u0001\u0000\u0000\u0000\u00d7\u00da\u0001"+ - "\u0000\u0000\u0000\u00d8\u00d6\u0001\u0000\u0000\u0000\u00d8\u00d9\u0001"+ - "\u0000\u0000\u0000\u00d9\u000b\u0001\u0000\u0000\u0000\u00da\u00d8\u0001"+ - "\u0000\u0000\u0000\u00db\u00dd\u0003\u0010\b\u0000\u00dc\u00de\u00051"+ - "\u0000\u0000\u00dd\u00dc\u0001\u0000\u0000\u0000\u00dd\u00de\u0001\u0000"+ - "\u0000\u0000\u00de\u00df\u0001\u0000\u0000\u0000\u00df\u00e0\u0005/\u0000"+ - "\u0000\u00e0\u00e1\u0003j5\u0000\u00e1\u00ea\u0001\u0000\u0000\u0000\u00e2"+ - "\u00e4\u0003\u0010\b\u0000\u00e3\u00e5\u00051\u0000\u0000\u00e4\u00e3"+ - "\u0001\u0000\u0000\u0000\u00e4\u00e5\u0001\u0000\u0000\u0000\u00e5\u00e6"+ - "\u0001\u0000\u0000\u0000\u00e6\u00e7\u00056\u0000\u0000\u00e7\u00e8\u0003"+ - "j5\u0000\u00e8\u00ea\u0001\u0000\u0000\u0000\u00e9\u00db\u0001\u0000\u0000"+ - "\u0000\u00e9\u00e2\u0001\u0000\u0000\u0000\u00ea\r\u0001\u0000\u0000\u0000"+ - "\u00eb\u00ee\u0003:\u001d\u0000\u00ec\u00ed\u0005%\u0000\u0000\u00ed\u00ef"+ - "\u0003\u001a\r\u0000\u00ee\u00ec\u0001\u0000\u0000\u0000\u00ee\u00ef\u0001"+ - "\u0000\u0000\u0000\u00ef\u00f0\u0001\u0000\u0000\u0000\u00f0\u00f1\u0005"+ - "&\u0000\u0000\u00f1\u00f2\u0003D\"\u0000\u00f2\u000f\u0001\u0000\u0000"+ - "\u0000\u00f3\u00f9\u0003\u0012\t\u0000\u00f4\u00f5\u0003\u0012\t\u0000"+ - "\u00f5\u00f6\u0003l6\u0000\u00f6\u00f7\u0003\u0012\t\u0000\u00f7\u00f9"+ - "\u0001\u0000\u0000\u0000\u00f8\u00f3\u0001\u0000\u0000\u0000\u00f8\u00f4"+ - "\u0001\u0000\u0000\u0000\u00f9\u0011\u0001\u0000\u0000\u0000\u00fa\u00fb"+ - "\u0006\t\uffff\uffff\u0000\u00fb\u00ff\u0003\u0014\n\u0000\u00fc\u00fd"+ - "\u0007\u0000\u0000\u0000\u00fd\u00ff\u0003\u0012\t\u0003\u00fe\u00fa\u0001"+ - "\u0000\u0000\u0000\u00fe\u00fc\u0001\u0000\u0000\u0000\u00ff\u0108\u0001"+ - "\u0000\u0000\u0000\u0100\u0101\n\u0002\u0000\u0000\u0101\u0102\u0007\u0001"+ - "\u0000\u0000\u0102\u0107\u0003\u0012\t\u0003\u0103\u0104\n\u0001\u0000"+ - "\u0000\u0104\u0105\u0007\u0000\u0000\u0000\u0105\u0107\u0003\u0012\t\u0002"+ - "\u0106\u0100\u0001\u0000\u0000\u0000\u0106\u0103\u0001\u0000\u0000\u0000"+ - "\u0107\u010a\u0001\u0000\u0000\u0000\u0108\u0106\u0001\u0000\u0000\u0000"+ - "\u0108\u0109\u0001\u0000\u0000\u0000\u0109\u0013\u0001\u0000\u0000\u0000"+ - "\u010a\u0108\u0001\u0000\u0000\u0000\u010b\u010c\u0006\n\uffff\uffff\u0000"+ - "\u010c\u0114\u0003D\"\u0000\u010d\u0114\u0003:\u001d\u0000\u010e\u0114"+ - "\u0003\u0016\u000b\u0000\u010f\u0110\u00050\u0000\u0000\u0110\u0111\u0003"+ - "\n\u0005\u0000\u0111\u0112\u00057\u0000\u0000\u0112\u0114\u0001\u0000"+ - "\u0000\u0000\u0113\u010b\u0001\u0000\u0000\u0000\u0113\u010d\u0001\u0000"+ - "\u0000\u0000\u0113\u010e\u0001\u0000\u0000\u0000\u0113\u010f\u0001\u0000"+ - "\u0000\u0000\u0114\u011a\u0001\u0000\u0000\u0000\u0115\u0116\n\u0001\u0000"+ - "\u0000\u0116\u0117\u0005%\u0000\u0000\u0117\u0119\u0003\u001a\r\u0000"+ - "\u0118\u0115\u0001\u0000\u0000\u0000\u0119\u011c\u0001\u0000\u0000\u0000"+ - "\u011a\u0118\u0001\u0000\u0000\u0000\u011a\u011b\u0001\u0000\u0000\u0000"+ - "\u011b\u0015\u0001\u0000\u0000\u0000\u011c\u011a\u0001\u0000\u0000\u0000"+ - "\u011d\u011e\u0003\u0018\f\u0000\u011e\u0128\u00050\u0000\u0000\u011f"+ - "\u0129\u0005B\u0000\u0000\u0120\u0125\u0003\n\u0005\u0000\u0121\u0122"+ - "\u0005\'\u0000\u0000\u0122\u0124\u0003\n\u0005\u0000\u0123\u0121\u0001"+ - "\u0000\u0000\u0000\u0124\u0127\u0001\u0000\u0000\u0000\u0125\u0123\u0001"+ - "\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126\u0129\u0001"+ - "\u0000\u0000\u0000\u0127\u0125\u0001\u0000\u0000\u0000\u0128\u011f\u0001"+ - "\u0000\u0000\u0000\u0128\u0120\u0001\u0000\u0000\u0000\u0128\u0129\u0001"+ - "\u0000\u0000\u0000\u0129\u012a\u0001\u0000\u0000\u0000\u012a\u012b\u0005"+ - "7\u0000\u0000\u012b\u0017\u0001\u0000\u0000\u0000\u012c\u012d\u0003H$"+ - "\u0000\u012d\u0019\u0001\u0000\u0000\u0000\u012e\u012f\u0003@ \u0000\u012f"+ - "\u001b\u0001\u0000\u0000\u0000\u0130\u0131\u0005\f\u0000\u0000\u0131\u0132"+ - "\u0003\u001e\u000f\u0000\u0132\u001d\u0001\u0000\u0000\u0000\u0133\u0138"+ - "\u0003 \u0010\u0000\u0134\u0135\u0005\'\u0000\u0000\u0135\u0137\u0003"+ - " \u0010\u0000\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u013a\u0001\u0000"+ - "\u0000\u0000\u0138\u0136\u0001\u0000\u0000\u0000\u0138\u0139\u0001\u0000"+ - "\u0000\u0000\u0139\u001f\u0001\u0000\u0000\u0000\u013a\u0138\u0001\u0000"+ - "\u0000\u0000\u013b\u013c\u0003:\u001d\u0000\u013c\u013d\u0005$\u0000\u0000"+ - "\u013d\u013f\u0001\u0000\u0000\u0000\u013e\u013b\u0001\u0000\u0000\u0000"+ - "\u013e\u013f\u0001\u0000\u0000\u0000\u013f\u0140\u0001\u0000\u0000\u0000"+ - "\u0140\u0141\u0003\n\u0005\u0000\u0141!\u0001\u0000\u0000\u0000\u0142"+ - "\u0143\u0005\u0006\u0000\u0000\u0143\u0148\u0003$\u0012\u0000\u0144\u0145"+ - "\u0005\'\u0000\u0000\u0145\u0147\u0003$\u0012\u0000\u0146\u0144\u0001"+ - "\u0000\u0000\u0000\u0147\u014a\u0001\u0000\u0000\u0000\u0148\u0146\u0001"+ - "\u0000\u0000\u0000\u0148\u0149\u0001\u0000\u0000\u0000\u0149\u014c\u0001"+ - "\u0000\u0000\u0000\u014a\u0148\u0001\u0000\u0000\u0000\u014b\u014d\u0003"+ - "*\u0015\u0000\u014c\u014b\u0001\u0000\u0000\u0000\u014c\u014d\u0001\u0000"+ - "\u0000\u0000\u014d#\u0001\u0000\u0000\u0000\u014e\u014f\u0003&\u0013\u0000"+ - "\u014f\u0150\u0005&\u0000\u0000\u0150\u0152\u0001\u0000\u0000\u0000\u0151"+ - "\u014e\u0001\u0000\u0000\u0000\u0151\u0152\u0001\u0000\u0000\u0000\u0152"+ - "\u0153\u0001\u0000\u0000\u0000\u0153\u0154\u0003(\u0014\u0000\u0154%\u0001"+ - "\u0000\u0000\u0000\u0155\u0156\u0005Q\u0000\u0000\u0156\'\u0001\u0000"+ - "\u0000\u0000\u0157\u0158\u0007\u0002\u0000\u0000\u0158)\u0001\u0000\u0000"+ - "\u0000\u0159\u015c\u0003,\u0016\u0000\u015a\u015c\u0003.\u0017\u0000\u015b"+ - "\u0159\u0001\u0000\u0000\u0000\u015b\u015a\u0001\u0000\u0000\u0000\u015c"+ - "+\u0001\u0000\u0000\u0000\u015d\u015e\u0005P\u0000\u0000\u015e\u0163\u0005"+ - "Q\u0000\u0000\u015f\u0160\u0005\'\u0000\u0000\u0160\u0162\u0005Q\u0000"+ - "\u0000\u0161\u015f\u0001\u0000\u0000\u0000\u0162\u0165\u0001\u0000\u0000"+ - "\u0000\u0163\u0161\u0001\u0000\u0000\u0000\u0163\u0164\u0001\u0000\u0000"+ - "\u0000\u0164-\u0001\u0000\u0000\u0000\u0165\u0163\u0001\u0000\u0000\u0000"+ - "\u0166\u0167\u0005F\u0000\u0000\u0167\u0168\u0003,\u0016\u0000\u0168\u0169"+ - "\u0005G\u0000\u0000\u0169/\u0001\u0000\u0000\u0000\u016a\u016b\u0005\u0013"+ - "\u0000\u0000\u016b\u0170\u0003$\u0012\u0000\u016c\u016d\u0005\'\u0000"+ - "\u0000\u016d\u016f\u0003$\u0012\u0000\u016e\u016c\u0001\u0000\u0000\u0000"+ - "\u016f\u0172\u0001\u0000\u0000\u0000\u0170\u016e\u0001\u0000\u0000\u0000"+ - "\u0170\u0171\u0001\u0000\u0000\u0000\u0171\u0174\u0001\u0000\u0000\u0000"+ - "\u0172\u0170\u0001\u0000\u0000\u0000\u0173\u0175\u00036\u001b\u0000\u0174"+ - "\u0173\u0001\u0000\u0000\u0000\u0174\u0175\u0001\u0000\u0000\u0000\u0175"+ - "\u0178\u0001\u0000\u0000\u0000\u0176\u0177\u0005!\u0000\u0000\u0177\u0179"+ - "\u0003\u001e\u000f\u0000\u0178\u0176\u0001\u0000\u0000\u0000\u0178\u0179"+ - "\u0001\u0000\u0000\u0000\u01791\u0001\u0000\u0000\u0000\u017a\u017b\u0005"+ - "\u0004\u0000\u0000\u017b\u017c\u0003\u001e\u000f\u0000\u017c3\u0001\u0000"+ - "\u0000\u0000\u017d\u017f\u0005\u000f\u0000\u0000\u017e\u0180\u00036\u001b"+ - "\u0000\u017f\u017e\u0001\u0000\u0000\u0000\u017f\u0180\u0001\u0000\u0000"+ - "\u0000\u0180\u0183\u0001\u0000\u0000\u0000\u0181\u0182\u0005!\u0000\u0000"+ - "\u0182\u0184\u0003\u001e\u000f\u0000\u0183\u0181\u0001\u0000\u0000\u0000"+ - "\u0183\u0184\u0001\u0000\u0000\u0000\u01845\u0001\u0000\u0000\u0000\u0185"+ - "\u018a\u00038\u001c\u0000\u0186\u0187\u0005\'\u0000\u0000\u0187\u0189"+ - "\u00038\u001c\u0000\u0188\u0186\u0001\u0000\u0000\u0000\u0189\u018c\u0001"+ - "\u0000\u0000\u0000\u018a\u0188\u0001\u0000\u0000\u0000\u018a\u018b\u0001"+ - "\u0000\u0000\u0000\u018b7\u0001\u0000\u0000\u0000\u018c\u018a\u0001\u0000"+ - "\u0000\u0000\u018d\u0190\u0003 \u0010\u0000\u018e\u018f\u0005\u0010\u0000"+ - "\u0000\u018f\u0191\u0003\n\u0005\u0000\u0190\u018e\u0001\u0000\u0000\u0000"+ - "\u0190\u0191\u0001\u0000\u0000\u0000\u01919\u0001\u0000\u0000\u0000\u0192"+ - "\u0197\u0003H$\u0000\u0193\u0194\u0005)\u0000\u0000\u0194\u0196\u0003"+ - "H$\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u0199\u0001\u0000\u0000"+ - "\u0000\u0197\u0195\u0001\u0000\u0000\u0000\u0197\u0198\u0001\u0000\u0000"+ - "\u0000\u0198;\u0001\u0000\u0000\u0000\u0199\u0197\u0001\u0000\u0000\u0000"+ - "\u019a\u019f\u0003B!\u0000\u019b\u019c\u0005)\u0000\u0000\u019c\u019e"+ - "\u0003B!\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e\u01a1\u0001\u0000"+ - "\u0000\u0000\u019f\u019d\u0001\u0000\u0000\u0000\u019f\u01a0\u0001\u0000"+ - "\u0000\u0000\u01a0=\u0001\u0000\u0000\u0000\u01a1\u019f\u0001\u0000\u0000"+ - "\u0000\u01a2\u01a7\u0003<\u001e\u0000\u01a3\u01a4\u0005\'\u0000\u0000"+ - "\u01a4\u01a6\u0003<\u001e\u0000\u01a5\u01a3\u0001\u0000\u0000\u0000\u01a6"+ - "\u01a9\u0001\u0000\u0000\u0000\u01a7\u01a5\u0001\u0000\u0000\u0000\u01a7"+ - "\u01a8\u0001\u0000\u0000\u0000\u01a8?\u0001\u0000\u0000\u0000\u01a9\u01a7"+ - "\u0001\u0000\u0000\u0000\u01aa\u01ab\u0007\u0003\u0000\u0000\u01abA\u0001"+ - "\u0000\u0000\u0000\u01ac\u01b0\u0005U\u0000\u0000\u01ad\u01ae\u0004!\n"+ - "\u0000\u01ae\u01b0\u0003F#\u0000\u01af\u01ac\u0001\u0000\u0000\u0000\u01af"+ - "\u01ad\u0001\u0000\u0000\u0000\u01b0C\u0001\u0000\u0000\u0000\u01b1\u01dc"+ - "\u00052\u0000\u0000\u01b2\u01b3\u0003h4\u0000\u01b3\u01b4\u0005H\u0000"+ - "\u0000\u01b4\u01dc\u0001\u0000\u0000\u0000\u01b5\u01dc\u0003f3\u0000\u01b6"+ - "\u01dc\u0003h4\u0000\u01b7\u01dc\u0003b1\u0000\u01b8\u01dc\u0003F#\u0000"+ - "\u01b9\u01dc\u0003j5\u0000\u01ba\u01bb\u0005F\u0000\u0000\u01bb\u01c0"+ - "\u0003d2\u0000\u01bc\u01bd\u0005\'\u0000\u0000\u01bd\u01bf\u0003d2\u0000"+ - "\u01be\u01bc\u0001\u0000\u0000\u0000\u01bf\u01c2\u0001\u0000\u0000\u0000"+ - "\u01c0\u01be\u0001\u0000\u0000\u0000\u01c0\u01c1\u0001\u0000\u0000\u0000"+ - "\u01c1\u01c3\u0001\u0000\u0000\u0000\u01c2\u01c0\u0001\u0000\u0000\u0000"+ - "\u01c3\u01c4\u0005G\u0000\u0000\u01c4\u01dc\u0001\u0000\u0000\u0000\u01c5"+ - "\u01c6\u0005F\u0000\u0000\u01c6\u01cb\u0003b1\u0000\u01c7\u01c8\u0005"+ - "\'\u0000\u0000\u01c8\u01ca\u0003b1\u0000\u01c9\u01c7\u0001\u0000\u0000"+ - "\u0000\u01ca\u01cd\u0001\u0000\u0000\u0000\u01cb\u01c9\u0001\u0000\u0000"+ - "\u0000\u01cb\u01cc\u0001\u0000\u0000\u0000\u01cc\u01ce\u0001\u0000\u0000"+ - "\u0000\u01cd\u01cb\u0001\u0000\u0000\u0000\u01ce\u01cf\u0005G\u0000\u0000"+ - "\u01cf\u01dc\u0001\u0000\u0000\u0000\u01d0\u01d1\u0005F\u0000\u0000\u01d1"+ - "\u01d6\u0003j5\u0000\u01d2\u01d3\u0005\'\u0000\u0000\u01d3\u01d5\u0003"+ - "j5\u0000\u01d4\u01d2\u0001\u0000\u0000\u0000\u01d5\u01d8\u0001\u0000\u0000"+ - "\u0000\u01d6\u01d4\u0001\u0000\u0000\u0000\u01d6\u01d7\u0001\u0000\u0000"+ - "\u0000\u01d7\u01d9\u0001\u0000\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000"+ - "\u0000\u01d9\u01da\u0005G\u0000\u0000\u01da\u01dc\u0001\u0000\u0000\u0000"+ - "\u01db\u01b1\u0001\u0000\u0000\u0000\u01db\u01b2\u0001\u0000\u0000\u0000"+ - "\u01db\u01b5\u0001\u0000\u0000\u0000\u01db\u01b6\u0001\u0000\u0000\u0000"+ - "\u01db\u01b7\u0001\u0000\u0000\u0000\u01db\u01b8\u0001\u0000\u0000\u0000"+ - "\u01db\u01b9\u0001\u0000\u0000\u0000\u01db\u01ba\u0001\u0000\u0000\u0000"+ - "\u01db\u01c5\u0001\u0000\u0000\u0000\u01db\u01d0\u0001\u0000\u0000\u0000"+ - "\u01dcE\u0001\u0000\u0000\u0000\u01dd\u01e0\u00055\u0000\u0000\u01de\u01e0"+ - "\u0005E\u0000\u0000\u01df\u01dd\u0001\u0000\u0000\u0000\u01df\u01de\u0001"+ - "\u0000\u0000\u0000\u01e0G\u0001\u0000\u0000\u0000\u01e1\u01e5\u0003@ "+ - "\u0000\u01e2\u01e3\u0004$\u000b\u0000\u01e3\u01e5\u0003F#\u0000\u01e4"+ - "\u01e1\u0001\u0000\u0000\u0000\u01e4\u01e2\u0001\u0000\u0000\u0000\u01e5"+ - "I\u0001\u0000\u0000\u0000\u01e6\u01e7\u0005\t\u0000\u0000\u01e7\u01e8"+ - "\u0005\u001f\u0000\u0000\u01e8K\u0001\u0000\u0000\u0000\u01e9\u01ea\u0005"+ - "\u000e\u0000\u0000\u01ea\u01ef\u0003N\'\u0000\u01eb\u01ec\u0005\'\u0000"+ - "\u0000\u01ec\u01ee\u0003N\'\u0000\u01ed\u01eb\u0001\u0000\u0000\u0000"+ - "\u01ee\u01f1\u0001\u0000\u0000\u0000\u01ef\u01ed\u0001\u0000\u0000\u0000"+ - "\u01ef\u01f0\u0001\u0000\u0000\u0000\u01f0M\u0001\u0000\u0000\u0000\u01f1"+ - "\u01ef\u0001\u0000\u0000\u0000\u01f2\u01f4\u0003\n\u0005\u0000\u01f3\u01f5"+ - "\u0007\u0004\u0000\u0000\u01f4\u01f3\u0001\u0000\u0000\u0000\u01f4\u01f5"+ - "\u0001\u0000\u0000\u0000\u01f5\u01f8\u0001\u0000\u0000\u0000\u01f6\u01f7"+ - "\u00053\u0000\u0000\u01f7\u01f9\u0007\u0005\u0000\u0000\u01f8\u01f6\u0001"+ - "\u0000\u0000\u0000\u01f8\u01f9\u0001\u0000\u0000\u0000\u01f9O\u0001\u0000"+ - "\u0000\u0000\u01fa\u01fb\u0005\b\u0000\u0000\u01fb\u01fc\u0003>\u001f"+ - "\u0000\u01fcQ\u0001\u0000\u0000\u0000\u01fd\u01fe\u0005\u0002\u0000\u0000"+ - "\u01fe\u01ff\u0003>\u001f\u0000\u01ffS\u0001\u0000\u0000\u0000\u0200\u0201"+ - "\u0005\u000b\u0000\u0000\u0201\u0206\u0003V+\u0000\u0202\u0203\u0005\'"+ - "\u0000\u0000\u0203\u0205\u0003V+\u0000\u0204\u0202\u0001\u0000\u0000\u0000"+ - "\u0205\u0208\u0001\u0000\u0000\u0000\u0206\u0204\u0001\u0000\u0000\u0000"+ - "\u0206\u0207\u0001\u0000\u0000\u0000\u0207U\u0001\u0000\u0000\u0000\u0208"+ - "\u0206\u0001\u0000\u0000\u0000\u0209\u020a\u0003<\u001e\u0000\u020a\u020b"+ - "\u0005Y\u0000\u0000\u020b\u020c\u0003<\u001e\u0000\u020cW\u0001\u0000"+ - "\u0000\u0000\u020d\u020e\u0005\u0001\u0000\u0000\u020e\u020f\u0003\u0014"+ - "\n\u0000\u020f\u0211\u0003j5\u0000\u0210\u0212\u0003^/\u0000\u0211\u0210"+ - "\u0001\u0000\u0000\u0000\u0211\u0212\u0001\u0000\u0000\u0000\u0212Y\u0001"+ - "\u0000\u0000\u0000\u0213\u0214\u0005\u0007\u0000\u0000\u0214\u0215\u0003"+ - "\u0014\n\u0000\u0215\u0216\u0003j5\u0000\u0216[\u0001\u0000\u0000\u0000"+ - "\u0217\u0218\u0005\n\u0000\u0000\u0218\u0219\u0003:\u001d\u0000\u0219"+ - "]\u0001\u0000\u0000\u0000\u021a\u021f\u0003`0\u0000\u021b\u021c\u0005"+ - "\'\u0000\u0000\u021c\u021e\u0003`0\u0000\u021d\u021b\u0001\u0000\u0000"+ - "\u0000\u021e\u0221\u0001\u0000\u0000\u0000\u021f\u021d\u0001\u0000\u0000"+ - "\u0000\u021f\u0220\u0001\u0000\u0000\u0000\u0220_\u0001\u0000\u0000\u0000"+ - "\u0221\u021f\u0001\u0000\u0000\u0000\u0222\u0223\u0003@ \u0000\u0223\u0224"+ - "\u0005$\u0000\u0000\u0224\u0225\u0003D\"\u0000\u0225a\u0001\u0000\u0000"+ - "\u0000\u0226\u0227\u0007\u0006\u0000\u0000\u0227c\u0001\u0000\u0000\u0000"+ - "\u0228\u022b\u0003f3\u0000\u0229\u022b\u0003h4\u0000\u022a\u0228\u0001"+ - "\u0000\u0000\u0000\u022a\u0229\u0001\u0000\u0000\u0000\u022be\u0001\u0000"+ - "\u0000\u0000\u022c\u022e\u0007\u0000\u0000\u0000\u022d\u022c\u0001\u0000"+ - "\u0000\u0000\u022d\u022e\u0001\u0000\u0000\u0000\u022e\u022f\u0001\u0000"+ - "\u0000\u0000\u022f\u0230\u0005 \u0000\u0000\u0230g\u0001\u0000\u0000\u0000"+ - "\u0231\u0233\u0007\u0000\u0000\u0000\u0232\u0231\u0001\u0000\u0000\u0000"+ - "\u0232\u0233\u0001\u0000\u0000\u0000\u0233\u0234\u0001\u0000\u0000\u0000"+ - "\u0234\u0235\u0005\u001f\u0000\u0000\u0235i\u0001\u0000\u0000\u0000\u0236"+ - "\u0237\u0005\u001e\u0000\u0000\u0237k\u0001\u0000\u0000\u0000\u0238\u0239"+ - "\u0007\u0007\u0000\u0000\u0239m\u0001\u0000\u0000\u0000\u023a\u023b\u0005"+ - "\u0005\u0000\u0000\u023b\u023c\u0003p8\u0000\u023co\u0001\u0000\u0000"+ - "\u0000\u023d\u023e\u0005F\u0000\u0000\u023e\u023f\u0003\u0002\u0001\u0000"+ - "\u023f\u0240\u0005G\u0000\u0000\u0240q\u0001\u0000\u0000\u0000\u0241\u0242"+ - "\u0005\r\u0000\u0000\u0242\u0243\u0005i\u0000\u0000\u0243s\u0001\u0000"+ - "\u0000\u0000\u0244\u0245\u0005\u0003\u0000\u0000\u0245\u0248\u0005_\u0000"+ - "\u0000\u0246\u0247\u0005]\u0000\u0000\u0247\u0249\u0003<\u001e\u0000\u0248"+ - "\u0246\u0001\u0000\u0000\u0000\u0248\u0249\u0001\u0000\u0000\u0000\u0249"+ - "\u0253\u0001\u0000\u0000\u0000\u024a\u024b\u0005^\u0000\u0000\u024b\u0250"+ - "\u0003v;\u0000\u024c\u024d\u0005\'\u0000\u0000\u024d\u024f\u0003v;\u0000"+ - "\u024e\u024c\u0001\u0000\u0000\u0000\u024f\u0252\u0001\u0000\u0000\u0000"+ - "\u0250\u024e\u0001\u0000\u0000\u0000\u0250\u0251\u0001\u0000\u0000\u0000"+ - "\u0251\u0254\u0001\u0000\u0000\u0000\u0252\u0250\u0001\u0000\u0000\u0000"+ - "\u0253\u024a\u0001\u0000\u0000\u0000\u0253\u0254\u0001\u0000\u0000\u0000"+ - "\u0254u\u0001\u0000\u0000\u0000\u0255\u0256\u0003<\u001e\u0000\u0256\u0257"+ - "\u0005$\u0000\u0000\u0257\u0259\u0001\u0000\u0000\u0000\u0258\u0255\u0001"+ - "\u0000\u0000\u0000\u0258\u0259\u0001\u0000\u0000\u0000\u0259\u025a\u0001"+ - "\u0000\u0000\u0000\u025a\u025b\u0003<\u001e\u0000\u025bw\u0001\u0000\u0000"+ - "\u0000\u025c\u025d\u0005\u0012\u0000\u0000\u025d\u025e\u0003$\u0012\u0000"+ - "\u025e\u025f\u0005]\u0000\u0000\u025f\u0260\u0003>\u001f\u0000\u0260y"+ - "\u0001\u0000\u0000\u0000\u0261\u0262\u0005\u0011\u0000\u0000\u0262\u0265"+ - "\u00036\u001b\u0000\u0263\u0264\u0005!\u0000\u0000\u0264\u0266\u0003\u001e"+ - "\u000f\u0000\u0265\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000"+ - "\u0000\u0000\u0266{\u0001\u0000\u0000\u0000\u0267\u0269\u0007\b\u0000"+ - "\u0000\u0268\u0267\u0001\u0000\u0000\u0000\u0268\u0269\u0001\u0000\u0000"+ - "\u0000\u0269\u026a\u0001\u0000\u0000\u0000\u026a\u026b\u0005\u0014\u0000"+ - "\u0000\u026b\u026c\u0003~?\u0000\u026c\u026d\u0003\u0080@\u0000\u026d"+ - "}\u0001\u0000\u0000\u0000\u026e\u0271\u0003@ \u0000\u026f\u0270\u0005"+ - "Y\u0000\u0000\u0270\u0272\u0003@ \u0000\u0271\u026f\u0001\u0000\u0000"+ - "\u0000\u0271\u0272\u0001\u0000\u0000\u0000\u0272\u007f\u0001\u0000\u0000"+ - "\u0000\u0273\u0274\u0005]\u0000\u0000\u0274\u0279\u0003\u0082A\u0000\u0275"+ - "\u0276\u0005\'\u0000\u0000\u0276\u0278\u0003\u0082A\u0000\u0277\u0275"+ - "\u0001\u0000\u0000\u0000\u0278\u027b\u0001\u0000\u0000\u0000\u0279\u0277"+ - "\u0001\u0000\u0000\u0000\u0279\u027a\u0001\u0000\u0000\u0000\u027a\u0081"+ - "\u0001\u0000\u0000\u0000\u027b\u0279\u0001\u0000\u0000\u0000\u027c\u027d"+ - "\u0003\u0010\b\u0000\u027d\u0083\u0001\u0000\u0000\u0000>\u008f\u0098"+ - "\u00ac\u00b8\u00c1\u00c9\u00ce\u00d6\u00d8\u00dd\u00e4\u00e9\u00ee\u00f8"+ - "\u00fe\u0106\u0108\u0113\u011a\u0125\u0128\u0138\u013e\u0148\u014c\u0151"+ - "\u015b\u0163\u0170\u0174\u0178\u017f\u0183\u018a\u0190\u0197\u019f\u01a7"+ - "\u01af\u01c0\u01cb\u01d6\u01db\u01df\u01e4\u01ef\u01f4\u01f8\u0206\u0211"+ - "\u021f\u022a\u022d\u0232\u0248\u0250\u0253\u0258\u0265\u0268\u0271\u0279"; + ";\u0002<\u0007<\u0002=\u0007=\u0002>\u0007>\u0002?\u0007?\u0001\u0000"+ + "\u0001\u0000\u0001\u0000\u0001\u0001\u0001\u0001\u0001\u0001\u0001\u0001"+ + "\u0001\u0001\u0001\u0001\u0005\u0001\u008a\b\u0001\n\u0001\f\u0001\u008d"+ + "\t\u0001\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001\u0002\u0001"+ + "\u0002\u0003\u0002\u0095\b\u0002\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001\u0003\u0001"+ + "\u0003\u0001\u0003\u0001\u0003\u0003\u0003\u00a9\b\u0003\u0001\u0004\u0001"+ + "\u0004\u0001\u0004\u0001\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0003\u0005\u00b5\b\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00bc\b\u0005\n"+ + "\u0005\f\u0005\u00bf\t\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0003\u0005\u00c6\b\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0003\u0005\u00cb\b\u0005\u0001\u0005\u0001\u0005\u0001\u0005\u0001"+ + "\u0005\u0001\u0005\u0001\u0005\u0005\u0005\u00d3\b\u0005\n\u0005\f\u0005"+ + "\u00d6\t\u0005\u0001\u0006\u0001\u0006\u0003\u0006\u00da\b\u0006\u0001"+ + "\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e1"+ + "\b\u0006\u0001\u0006\u0001\u0006\u0001\u0006\u0003\u0006\u00e6\b\u0006"+ + "\u0001\u0007\u0001\u0007\u0001\u0007\u0003\u0007\u00eb\b\u0007\u0001\u0007"+ + "\u0001\u0007\u0001\u0007\u0001\b\u0001\b\u0001\b\u0001\b\u0001\b\u0003"+ + "\b\u00f5\b\b\u0001\t\u0001\t\u0001\t\u0001\t\u0003\t\u00fb\b\t\u0001\t"+ + "\u0001\t\u0001\t\u0001\t\u0001\t\u0001\t\u0005\t\u0103\b\t\n\t\f\t\u0106"+ + "\t\t\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0001\n\u0003"+ + "\n\u0110\b\n\u0001\n\u0001\n\u0001\n\u0005\n\u0115\b\n\n\n\f\n\u0118\t"+ + "\n\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b\u0001\u000b"+ + "\u0005\u000b\u0120\b\u000b\n\u000b\f\u000b\u0123\t\u000b\u0003\u000b\u0125"+ + "\b\u000b\u0001\u000b\u0001\u000b\u0001\f\u0001\f\u0001\r\u0001\r\u0001"+ + "\u000e\u0001\u000e\u0001\u000e\u0001\u000f\u0001\u000f\u0001\u000f\u0005"+ + "\u000f\u0133\b\u000f\n\u000f\f\u000f\u0136\t\u000f\u0001\u0010\u0001\u0010"+ + "\u0001\u0010\u0003\u0010\u013b\b\u0010\u0001\u0010\u0001\u0010\u0001\u0011"+ + "\u0001\u0011\u0001\u0011\u0001\u0011\u0005\u0011\u0143\b\u0011\n\u0011"+ + "\f\u0011\u0146\t\u0011\u0001\u0011\u0003\u0011\u0149\b\u0011\u0001\u0012"+ + "\u0001\u0012\u0001\u0012\u0003\u0012\u014e\b\u0012\u0001\u0012\u0001\u0012"+ + "\u0001\u0013\u0001\u0013\u0001\u0014\u0001\u0014\u0001\u0015\u0001\u0015"+ + "\u0001\u0015\u0001\u0015\u0005\u0015\u015a\b\u0015\n\u0015\f\u0015\u015d"+ + "\t\u0015\u0001\u0016\u0001\u0016\u0001\u0016\u0001\u0016\u0005\u0016\u0163"+ + "\b\u0016\n\u0016\f\u0016\u0166\t\u0016\u0001\u0016\u0003\u0016\u0169\b"+ + "\u0016\u0001\u0016\u0001\u0016\u0003\u0016\u016d\b\u0016\u0001\u0017\u0001"+ + "\u0017\u0001\u0017\u0001\u0018\u0001\u0018\u0003\u0018\u0174\b\u0018\u0001"+ + "\u0018\u0001\u0018\u0003\u0018\u0178\b\u0018\u0001\u0019\u0001\u0019\u0001"+ + "\u0019\u0005\u0019\u017d\b\u0019\n\u0019\f\u0019\u0180\t\u0019\u0001\u001a"+ + "\u0001\u001a\u0001\u001a\u0003\u001a\u0185\b\u001a\u0001\u001b\u0001\u001b"+ + "\u0001\u001b\u0005\u001b\u018a\b\u001b\n\u001b\f\u001b\u018d\t\u001b\u0001"+ + "\u001c\u0001\u001c\u0001\u001c\u0005\u001c\u0192\b\u001c\n\u001c\f\u001c"+ + "\u0195\t\u001c\u0001\u001d\u0001\u001d\u0001\u001d\u0005\u001d\u019a\b"+ + "\u001d\n\u001d\f\u001d\u019d\t\u001d\u0001\u001e\u0001\u001e\u0001\u001f"+ + "\u0001\u001f\u0001\u001f\u0003\u001f\u01a4\b\u001f\u0001 \u0001 \u0001"+ + " \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01b3\b \n \f \u01b6\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01be\b \n \f \u01c1\t \u0001 \u0001 \u0001 \u0001 \u0001 \u0001"+ + " \u0005 \u01c9\b \n \f \u01cc\t \u0001 \u0001 \u0003 \u01d0\b \u0001!"+ + "\u0001!\u0003!\u01d4\b!\u0001\"\u0001\"\u0001\"\u0003\"\u01d9\b\"\u0001"+ + "#\u0001#\u0001#\u0001$\u0001$\u0001$\u0001$\u0005$\u01e2\b$\n$\f$\u01e5"+ + "\t$\u0001%\u0001%\u0003%\u01e9\b%\u0001%\u0001%\u0003%\u01ed\b%\u0001"+ + "&\u0001&\u0001&\u0001\'\u0001\'\u0001\'\u0001(\u0001(\u0001(\u0001(\u0005"+ + "(\u01f9\b(\n(\f(\u01fc\t(\u0001)\u0001)\u0001)\u0001)\u0001*\u0001*\u0001"+ + "*\u0001*\u0003*\u0206\b*\u0001+\u0001+\u0001+\u0001+\u0001,\u0001,\u0001"+ + ",\u0001-\u0001-\u0001-\u0005-\u0212\b-\n-\f-\u0215\t-\u0001.\u0001.\u0001"+ + ".\u0001.\u0001/\u0001/\u00010\u00010\u00030\u021f\b0\u00011\u00031\u0222"+ + "\b1\u00011\u00011\u00012\u00032\u0227\b2\u00012\u00012\u00013\u00013\u0001"+ + "4\u00014\u00015\u00015\u00015\u00016\u00016\u00016\u00016\u00017\u0001"+ + "7\u00017\u00018\u00018\u00018\u00018\u00038\u023d\b8\u00018\u00018\u0001"+ + "8\u00018\u00058\u0243\b8\n8\f8\u0246\t8\u00038\u0248\b8\u00019\u00019"+ + "\u00019\u00039\u024d\b9\u00019\u00019\u0001:\u0001:\u0001:\u0001:\u0001"+ + ":\u0001;\u0001;\u0001;\u0001;\u0003;\u025a\b;\u0001<\u0003<\u025d\b<\u0001"+ + "<\u0001<\u0001<\u0001<\u0001=\u0001=\u0001=\u0003=\u0266\b=\u0001>\u0001"+ + ">\u0001>\u0001>\u0005>\u026c\b>\n>\f>\u026f\t>\u0001?\u0001?\u0001?\u0000"+ + "\u0004\u0002\n\u0012\u0014@\u0000\u0002\u0004\u0006\b\n\f\u000e\u0010"+ + "\u0012\u0014\u0016\u0018\u001a\u001c\u001e \"$&(*,.02468:<>@BDFHJLNPR"+ + "TVXZ\\^`bdfhjlnprtvxz|~\u0000\t\u0001\u0000@A\u0001\u0000BD\u0002\u0000"+ + "\u001e\u001eQQ\u0001\u0000HI\u0002\u0000##((\u0002\u0000++..\u0002\u0000"+ + "**88\u0002\u000099;?\u0001\u0000\u0016\u0018\u028e\u0000\u0080\u0001\u0000"+ + "\u0000\u0000\u0002\u0083\u0001\u0000\u0000\u0000\u0004\u0094\u0001\u0000"+ + "\u0000\u0000\u0006\u00a8\u0001\u0000\u0000\u0000\b\u00aa\u0001\u0000\u0000"+ + "\u0000\n\u00ca\u0001\u0000\u0000\u0000\f\u00e5\u0001\u0000\u0000\u0000"+ + "\u000e\u00e7\u0001\u0000\u0000\u0000\u0010\u00f4\u0001\u0000\u0000\u0000"+ + "\u0012\u00fa\u0001\u0000\u0000\u0000\u0014\u010f\u0001\u0000\u0000\u0000"+ + "\u0016\u0119\u0001\u0000\u0000\u0000\u0018\u0128\u0001\u0000\u0000\u0000"+ + "\u001a\u012a\u0001\u0000\u0000\u0000\u001c\u012c\u0001\u0000\u0000\u0000"+ + "\u001e\u012f\u0001\u0000\u0000\u0000 \u013a\u0001\u0000\u0000\u0000\""+ + "\u013e\u0001\u0000\u0000\u0000$\u014d\u0001\u0000\u0000\u0000&\u0151\u0001"+ + "\u0000\u0000\u0000(\u0153\u0001\u0000\u0000\u0000*\u0155\u0001\u0000\u0000"+ + "\u0000,\u015e\u0001\u0000\u0000\u0000.\u016e\u0001\u0000\u0000\u00000"+ + "\u0171\u0001\u0000\u0000\u00002\u0179\u0001\u0000\u0000\u00004\u0181\u0001"+ + "\u0000\u0000\u00006\u0186\u0001\u0000\u0000\u00008\u018e\u0001\u0000\u0000"+ + "\u0000:\u0196\u0001\u0000\u0000\u0000<\u019e\u0001\u0000\u0000\u0000>"+ + "\u01a3\u0001\u0000\u0000\u0000@\u01cf\u0001\u0000\u0000\u0000B\u01d3\u0001"+ + "\u0000\u0000\u0000D\u01d8\u0001\u0000\u0000\u0000F\u01da\u0001\u0000\u0000"+ + "\u0000H\u01dd\u0001\u0000\u0000\u0000J\u01e6\u0001\u0000\u0000\u0000L"+ + "\u01ee\u0001\u0000\u0000\u0000N\u01f1\u0001\u0000\u0000\u0000P\u01f4\u0001"+ + "\u0000\u0000\u0000R\u01fd\u0001\u0000\u0000\u0000T\u0201\u0001\u0000\u0000"+ + "\u0000V\u0207\u0001\u0000\u0000\u0000X\u020b\u0001\u0000\u0000\u0000Z"+ + "\u020e\u0001\u0000\u0000\u0000\\\u0216\u0001\u0000\u0000\u0000^\u021a"+ + "\u0001\u0000\u0000\u0000`\u021e\u0001\u0000\u0000\u0000b\u0221\u0001\u0000"+ + "\u0000\u0000d\u0226\u0001\u0000\u0000\u0000f\u022a\u0001\u0000\u0000\u0000"+ + "h\u022c\u0001\u0000\u0000\u0000j\u022e\u0001\u0000\u0000\u0000l\u0231"+ + "\u0001\u0000\u0000\u0000n\u0235\u0001\u0000\u0000\u0000p\u0238\u0001\u0000"+ + "\u0000\u0000r\u024c\u0001\u0000\u0000\u0000t\u0250\u0001\u0000\u0000\u0000"+ + "v\u0255\u0001\u0000\u0000\u0000x\u025c\u0001\u0000\u0000\u0000z\u0262"+ + "\u0001\u0000\u0000\u0000|\u0267\u0001\u0000\u0000\u0000~\u0270\u0001\u0000"+ + "\u0000\u0000\u0080\u0081\u0003\u0002\u0001\u0000\u0081\u0082\u0005\u0000"+ + "\u0000\u0001\u0082\u0001\u0001\u0000\u0000\u0000\u0083\u0084\u0006\u0001"+ + "\uffff\uffff\u0000\u0084\u0085\u0003\u0004\u0002\u0000\u0085\u008b\u0001"+ + "\u0000\u0000\u0000\u0086\u0087\n\u0001\u0000\u0000\u0087\u0088\u0005\u001d"+ + "\u0000\u0000\u0088\u008a\u0003\u0006\u0003\u0000\u0089\u0086\u0001\u0000"+ + "\u0000\u0000\u008a\u008d\u0001\u0000\u0000\u0000\u008b\u0089\u0001\u0000"+ + "\u0000\u0000\u008b\u008c\u0001\u0000\u0000\u0000\u008c\u0003\u0001\u0000"+ + "\u0000\u0000\u008d\u008b\u0001\u0000\u0000\u0000\u008e\u0095\u0003j5\u0000"+ + "\u008f\u0095\u0003\"\u0011\u0000\u0090\u0095\u0003\u001c\u000e\u0000\u0091"+ + "\u0095\u0003n7\u0000\u0092\u0093\u0004\u0002\u0001\u0000\u0093\u0095\u0003"+ + ",\u0016\u0000\u0094\u008e\u0001\u0000\u0000\u0000\u0094\u008f\u0001\u0000"+ + "\u0000\u0000\u0094\u0090\u0001\u0000\u0000\u0000\u0094\u0091\u0001\u0000"+ + "\u0000\u0000\u0094\u0092\u0001\u0000\u0000\u0000\u0095\u0005\u0001\u0000"+ + "\u0000\u0000\u0096\u00a9\u0003.\u0017\u0000\u0097\u00a9\u0003\b\u0004"+ + "\u0000\u0098\u00a9\u0003L&\u0000\u0099\u00a9\u0003F#\u0000\u009a\u00a9"+ + "\u00030\u0018\u0000\u009b\u00a9\u0003H$\u0000\u009c\u00a9\u0003N\'\u0000"+ + "\u009d\u00a9\u0003P(\u0000\u009e\u00a9\u0003T*\u0000\u009f\u00a9\u0003"+ + "V+\u0000\u00a0\u00a9\u0003p8\u0000\u00a1\u00a9\u0003X,\u0000\u00a2\u00a3"+ + "\u0004\u0003\u0002\u0000\u00a3\u00a9\u0003v;\u0000\u00a4\u00a5\u0004\u0003"+ + "\u0003\u0000\u00a5\u00a9\u0003t:\u0000\u00a6\u00a7\u0004\u0003\u0004\u0000"+ + "\u00a7\u00a9\u0003x<\u0000\u00a8\u0096\u0001\u0000\u0000\u0000\u00a8\u0097"+ + "\u0001\u0000\u0000\u0000\u00a8\u0098\u0001\u0000\u0000\u0000\u00a8\u0099"+ + "\u0001\u0000\u0000\u0000\u00a8\u009a\u0001\u0000\u0000\u0000\u00a8\u009b"+ + "\u0001\u0000\u0000\u0000\u00a8\u009c\u0001\u0000\u0000\u0000\u00a8\u009d"+ + "\u0001\u0000\u0000\u0000\u00a8\u009e\u0001\u0000\u0000\u0000\u00a8\u009f"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a0\u0001\u0000\u0000\u0000\u00a8\u00a1"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a2\u0001\u0000\u0000\u0000\u00a8\u00a4"+ + "\u0001\u0000\u0000\u0000\u00a8\u00a6\u0001\u0000\u0000\u0000\u00a9\u0007"+ + "\u0001\u0000\u0000\u0000\u00aa\u00ab\u0005\u0010\u0000\u0000\u00ab\u00ac"+ + "\u0003\n\u0005\u0000\u00ac\t\u0001\u0000\u0000\u0000\u00ad\u00ae\u0006"+ + "\u0005\uffff\uffff\u0000\u00ae\u00af\u00051\u0000\u0000\u00af\u00cb\u0003"+ + "\n\u0005\b\u00b0\u00cb\u0003\u0010\b\u0000\u00b1\u00cb\u0003\f\u0006\u0000"+ + "\u00b2\u00b4\u0003\u0010\b\u0000\u00b3\u00b5\u00051\u0000\u0000\u00b4"+ + "\u00b3\u0001\u0000\u0000\u0000\u00b4\u00b5\u0001\u0000\u0000\u0000\u00b5"+ + "\u00b6\u0001\u0000\u0000\u0000\u00b6\u00b7\u0005,\u0000\u0000\u00b7\u00b8"+ + "\u00050\u0000\u0000\u00b8\u00bd\u0003\u0010\b\u0000\u00b9\u00ba\u0005"+ + "\'\u0000\u0000\u00ba\u00bc\u0003\u0010\b\u0000\u00bb\u00b9\u0001\u0000"+ + "\u0000\u0000\u00bc\u00bf\u0001\u0000\u0000\u0000\u00bd\u00bb\u0001\u0000"+ + "\u0000\u0000\u00bd\u00be\u0001\u0000\u0000\u0000\u00be\u00c0\u0001\u0000"+ + "\u0000\u0000\u00bf\u00bd\u0001\u0000\u0000\u0000\u00c0\u00c1\u00057\u0000"+ + "\u0000\u00c1\u00cb\u0001\u0000\u0000\u0000\u00c2\u00c3\u0003\u0010\b\u0000"+ + "\u00c3\u00c5\u0005-\u0000\u0000\u00c4\u00c6\u00051\u0000\u0000\u00c5\u00c4"+ + "\u0001\u0000\u0000\u0000\u00c5\u00c6\u0001\u0000\u0000\u0000\u00c6\u00c7"+ + "\u0001\u0000\u0000\u0000\u00c7\u00c8\u00052\u0000\u0000\u00c8\u00cb\u0001"+ + "\u0000\u0000\u0000\u00c9\u00cb\u0003\u000e\u0007\u0000\u00ca\u00ad\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b0\u0001\u0000\u0000\u0000\u00ca\u00b1\u0001"+ + "\u0000\u0000\u0000\u00ca\u00b2\u0001\u0000\u0000\u0000\u00ca\u00c2\u0001"+ + "\u0000\u0000\u0000\u00ca\u00c9\u0001\u0000\u0000\u0000\u00cb\u00d4\u0001"+ + "\u0000\u0000\u0000\u00cc\u00cd\n\u0005\u0000\u0000\u00cd\u00ce\u0005\""+ + "\u0000\u0000\u00ce\u00d3\u0003\n\u0005\u0006\u00cf\u00d0\n\u0004\u0000"+ + "\u0000\u00d0\u00d1\u00054\u0000\u0000\u00d1\u00d3\u0003\n\u0005\u0005"+ + "\u00d2\u00cc\u0001\u0000\u0000\u0000\u00d2\u00cf\u0001\u0000\u0000\u0000"+ + "\u00d3\u00d6\u0001\u0000\u0000\u0000\u00d4\u00d2\u0001\u0000\u0000\u0000"+ + "\u00d4\u00d5\u0001\u0000\u0000\u0000\u00d5\u000b\u0001\u0000\u0000\u0000"+ + "\u00d6\u00d4\u0001\u0000\u0000\u0000\u00d7\u00d9\u0003\u0010\b\u0000\u00d8"+ + "\u00da\u00051\u0000\u0000\u00d9\u00d8\u0001\u0000\u0000\u0000\u00d9\u00da"+ + "\u0001\u0000\u0000\u0000\u00da\u00db\u0001\u0000\u0000\u0000\u00db\u00dc"+ + "\u0005/\u0000\u0000\u00dc\u00dd\u0003f3\u0000\u00dd\u00e6\u0001\u0000"+ + "\u0000\u0000\u00de\u00e0\u0003\u0010\b\u0000\u00df\u00e1\u00051\u0000"+ + "\u0000\u00e0\u00df\u0001\u0000\u0000\u0000\u00e0\u00e1\u0001\u0000\u0000"+ + "\u0000\u00e1\u00e2\u0001\u0000\u0000\u0000\u00e2\u00e3\u00056\u0000\u0000"+ + "\u00e3\u00e4\u0003f3\u0000\u00e4\u00e6\u0001\u0000\u0000\u0000\u00e5\u00d7"+ + "\u0001\u0000\u0000\u0000\u00e5\u00de\u0001\u0000\u0000\u0000\u00e6\r\u0001"+ + "\u0000\u0000\u0000\u00e7\u00ea\u00036\u001b\u0000\u00e8\u00e9\u0005%\u0000"+ + "\u0000\u00e9\u00eb\u0003\u001a\r\u0000\u00ea\u00e8\u0001\u0000\u0000\u0000"+ + "\u00ea\u00eb\u0001\u0000\u0000\u0000\u00eb\u00ec\u0001\u0000\u0000\u0000"+ + "\u00ec\u00ed\u0005&\u0000\u0000\u00ed\u00ee\u0003@ \u0000\u00ee\u000f"+ + "\u0001\u0000\u0000\u0000\u00ef\u00f5\u0003\u0012\t\u0000\u00f0\u00f1\u0003"+ + "\u0012\t\u0000\u00f1\u00f2\u0003h4\u0000\u00f2\u00f3\u0003\u0012\t\u0000"+ + "\u00f3\u00f5\u0001\u0000\u0000\u0000\u00f4\u00ef\u0001\u0000\u0000\u0000"+ + "\u00f4\u00f0\u0001\u0000\u0000\u0000\u00f5\u0011\u0001\u0000\u0000\u0000"+ + "\u00f6\u00f7\u0006\t\uffff\uffff\u0000\u00f7\u00fb\u0003\u0014\n\u0000"+ + "\u00f8\u00f9\u0007\u0000\u0000\u0000\u00f9\u00fb\u0003\u0012\t\u0003\u00fa"+ + "\u00f6\u0001\u0000\u0000\u0000\u00fa\u00f8\u0001\u0000\u0000\u0000\u00fb"+ + "\u0104\u0001\u0000\u0000\u0000\u00fc\u00fd\n\u0002\u0000\u0000\u00fd\u00fe"+ + "\u0007\u0001\u0000\u0000\u00fe\u0103\u0003\u0012\t\u0003\u00ff\u0100\n"+ + "\u0001\u0000\u0000\u0100\u0101\u0007\u0000\u0000\u0000\u0101\u0103\u0003"+ + "\u0012\t\u0002\u0102\u00fc\u0001\u0000\u0000\u0000\u0102\u00ff\u0001\u0000"+ + "\u0000\u0000\u0103\u0106\u0001\u0000\u0000\u0000\u0104\u0102\u0001\u0000"+ + "\u0000\u0000\u0104\u0105\u0001\u0000\u0000\u0000\u0105\u0013\u0001\u0000"+ + "\u0000\u0000\u0106\u0104\u0001\u0000\u0000\u0000\u0107\u0108\u0006\n\uffff"+ + "\uffff\u0000\u0108\u0110\u0003@ \u0000\u0109\u0110\u00036\u001b\u0000"+ + "\u010a\u0110\u0003\u0016\u000b\u0000\u010b\u010c\u00050\u0000\u0000\u010c"+ + "\u010d\u0003\n\u0005\u0000\u010d\u010e\u00057\u0000\u0000\u010e\u0110"+ + "\u0001\u0000\u0000\u0000\u010f\u0107\u0001\u0000\u0000\u0000\u010f\u0109"+ + "\u0001\u0000\u0000\u0000\u010f\u010a\u0001\u0000\u0000\u0000\u010f\u010b"+ + "\u0001\u0000\u0000\u0000\u0110\u0116\u0001\u0000\u0000\u0000\u0111\u0112"+ + "\n\u0001\u0000\u0000\u0112\u0113\u0005%\u0000\u0000\u0113\u0115\u0003"+ + "\u001a\r\u0000\u0114\u0111\u0001\u0000\u0000\u0000\u0115\u0118\u0001\u0000"+ + "\u0000\u0000\u0116\u0114\u0001\u0000\u0000\u0000\u0116\u0117\u0001\u0000"+ + "\u0000\u0000\u0117\u0015\u0001\u0000\u0000\u0000\u0118\u0116\u0001\u0000"+ + "\u0000\u0000\u0119\u011a\u0003\u0018\f\u0000\u011a\u0124\u00050\u0000"+ + "\u0000\u011b\u0125\u0005B\u0000\u0000\u011c\u0121\u0003\n\u0005\u0000"+ + "\u011d\u011e\u0005\'\u0000\u0000\u011e\u0120\u0003\n\u0005\u0000\u011f"+ + "\u011d\u0001\u0000\u0000\u0000\u0120\u0123\u0001\u0000\u0000\u0000\u0121"+ + "\u011f\u0001\u0000\u0000\u0000\u0121\u0122\u0001\u0000\u0000\u0000\u0122"+ + "\u0125\u0001\u0000\u0000\u0000\u0123\u0121\u0001\u0000\u0000\u0000\u0124"+ + "\u011b\u0001\u0000\u0000\u0000\u0124\u011c\u0001\u0000\u0000\u0000\u0124"+ + "\u0125\u0001\u0000\u0000\u0000\u0125\u0126\u0001\u0000\u0000\u0000\u0126"+ + "\u0127\u00057\u0000\u0000\u0127\u0017\u0001\u0000\u0000\u0000\u0128\u0129"+ + "\u0003D\"\u0000\u0129\u0019\u0001\u0000\u0000\u0000\u012a\u012b\u0003"+ + "<\u001e\u0000\u012b\u001b\u0001\u0000\u0000\u0000\u012c\u012d\u0005\f"+ + "\u0000\u0000\u012d\u012e\u0003\u001e\u000f\u0000\u012e\u001d\u0001\u0000"+ + "\u0000\u0000\u012f\u0134\u0003 \u0010\u0000\u0130\u0131\u0005\'\u0000"+ + "\u0000\u0131\u0133\u0003 \u0010\u0000\u0132\u0130\u0001\u0000\u0000\u0000"+ + "\u0133\u0136\u0001\u0000\u0000\u0000\u0134\u0132\u0001\u0000\u0000\u0000"+ + "\u0134\u0135\u0001\u0000\u0000\u0000\u0135\u001f\u0001\u0000\u0000\u0000"+ + "\u0136\u0134\u0001\u0000\u0000\u0000\u0137\u0138\u00036\u001b\u0000\u0138"+ + "\u0139\u0005$\u0000\u0000\u0139\u013b\u0001\u0000\u0000\u0000\u013a\u0137"+ + "\u0001\u0000\u0000\u0000\u013a\u013b\u0001\u0000\u0000\u0000\u013b\u013c"+ + "\u0001\u0000\u0000\u0000\u013c\u013d\u0003\n\u0005\u0000\u013d!\u0001"+ + "\u0000\u0000\u0000\u013e\u013f\u0005\u0006\u0000\u0000\u013f\u0144\u0003"+ + "$\u0012\u0000\u0140\u0141\u0005\'\u0000\u0000\u0141\u0143\u0003$\u0012"+ + "\u0000\u0142\u0140\u0001\u0000\u0000\u0000\u0143\u0146\u0001\u0000\u0000"+ + "\u0000\u0144\u0142\u0001\u0000\u0000\u0000\u0144\u0145\u0001\u0000\u0000"+ + "\u0000\u0145\u0148\u0001\u0000\u0000\u0000\u0146\u0144\u0001\u0000\u0000"+ + "\u0000\u0147\u0149\u0003*\u0015\u0000\u0148\u0147\u0001\u0000\u0000\u0000"+ + "\u0148\u0149\u0001\u0000\u0000\u0000\u0149#\u0001\u0000\u0000\u0000\u014a"+ + "\u014b\u0003&\u0013\u0000\u014b\u014c\u0005&\u0000\u0000\u014c\u014e\u0001"+ + "\u0000\u0000\u0000\u014d\u014a\u0001\u0000\u0000\u0000\u014d\u014e\u0001"+ + "\u0000\u0000\u0000\u014e\u014f\u0001\u0000\u0000\u0000\u014f\u0150\u0003"+ + "(\u0014\u0000\u0150%\u0001\u0000\u0000\u0000\u0151\u0152\u0005Q\u0000"+ + "\u0000\u0152\'\u0001\u0000\u0000\u0000\u0153\u0154\u0007\u0002\u0000\u0000"+ + "\u0154)\u0001\u0000\u0000\u0000\u0155\u0156\u0005P\u0000\u0000\u0156\u015b"+ + "\u0005Q\u0000\u0000\u0157\u0158\u0005\'\u0000\u0000\u0158\u015a\u0005"+ + "Q\u0000\u0000\u0159\u0157\u0001\u0000\u0000\u0000\u015a\u015d\u0001\u0000"+ + "\u0000\u0000\u015b\u0159\u0001\u0000\u0000\u0000\u015b\u015c\u0001\u0000"+ + "\u0000\u0000\u015c+\u0001\u0000\u0000\u0000\u015d\u015b\u0001\u0000\u0000"+ + "\u0000\u015e\u015f\u0005\u0013\u0000\u0000\u015f\u0164\u0003$\u0012\u0000"+ + "\u0160\u0161\u0005\'\u0000\u0000\u0161\u0163\u0003$\u0012\u0000\u0162"+ + "\u0160\u0001\u0000\u0000\u0000\u0163\u0166\u0001\u0000\u0000\u0000\u0164"+ + "\u0162\u0001\u0000\u0000\u0000\u0164\u0165\u0001\u0000\u0000\u0000\u0165"+ + "\u0168\u0001\u0000\u0000\u0000\u0166\u0164\u0001\u0000\u0000\u0000\u0167"+ + "\u0169\u00032\u0019\u0000\u0168\u0167\u0001\u0000\u0000\u0000\u0168\u0169"+ + "\u0001\u0000\u0000\u0000\u0169\u016c\u0001\u0000\u0000\u0000\u016a\u016b"+ + "\u0005!\u0000\u0000\u016b\u016d\u0003\u001e\u000f\u0000\u016c\u016a\u0001"+ + "\u0000\u0000\u0000\u016c\u016d\u0001\u0000\u0000\u0000\u016d-\u0001\u0000"+ + "\u0000\u0000\u016e\u016f\u0005\u0004\u0000\u0000\u016f\u0170\u0003\u001e"+ + "\u000f\u0000\u0170/\u0001\u0000\u0000\u0000\u0171\u0173\u0005\u000f\u0000"+ + "\u0000\u0172\u0174\u00032\u0019\u0000\u0173\u0172\u0001\u0000\u0000\u0000"+ + "\u0173\u0174\u0001\u0000\u0000\u0000\u0174\u0177\u0001\u0000\u0000\u0000"+ + "\u0175\u0176\u0005!\u0000\u0000\u0176\u0178\u0003\u001e\u000f\u0000\u0177"+ + "\u0175\u0001\u0000\u0000\u0000\u0177\u0178\u0001\u0000\u0000\u0000\u0178"+ + "1\u0001\u0000\u0000\u0000\u0179\u017e\u00034\u001a\u0000\u017a\u017b\u0005"+ + "\'\u0000\u0000\u017b\u017d\u00034\u001a\u0000\u017c\u017a\u0001\u0000"+ + "\u0000\u0000\u017d\u0180\u0001\u0000\u0000\u0000\u017e\u017c\u0001\u0000"+ + "\u0000\u0000\u017e\u017f\u0001\u0000\u0000\u0000\u017f3\u0001\u0000\u0000"+ + "\u0000\u0180\u017e\u0001\u0000\u0000\u0000\u0181\u0184\u0003 \u0010\u0000"+ + "\u0182\u0183\u0005\u0010\u0000\u0000\u0183\u0185\u0003\n\u0005\u0000\u0184"+ + "\u0182\u0001\u0000\u0000\u0000\u0184\u0185\u0001\u0000\u0000\u0000\u0185"+ + "5\u0001\u0000\u0000\u0000\u0186\u018b\u0003D\"\u0000\u0187\u0188\u0005"+ + ")\u0000\u0000\u0188\u018a\u0003D\"\u0000\u0189\u0187\u0001\u0000\u0000"+ + "\u0000\u018a\u018d\u0001\u0000\u0000\u0000\u018b\u0189\u0001\u0000\u0000"+ + "\u0000\u018b\u018c\u0001\u0000\u0000\u0000\u018c7\u0001\u0000\u0000\u0000"+ + "\u018d\u018b\u0001\u0000\u0000\u0000\u018e\u0193\u0003>\u001f\u0000\u018f"+ + "\u0190\u0005)\u0000\u0000\u0190\u0192\u0003>\u001f\u0000\u0191\u018f\u0001"+ + "\u0000\u0000\u0000\u0192\u0195\u0001\u0000\u0000\u0000\u0193\u0191\u0001"+ + "\u0000\u0000\u0000\u0193\u0194\u0001\u0000\u0000\u0000\u01949\u0001\u0000"+ + "\u0000\u0000\u0195\u0193\u0001\u0000\u0000\u0000\u0196\u019b\u00038\u001c"+ + "\u0000\u0197\u0198\u0005\'\u0000\u0000\u0198\u019a\u00038\u001c\u0000"+ + "\u0199\u0197\u0001\u0000\u0000\u0000\u019a\u019d\u0001\u0000\u0000\u0000"+ + "\u019b\u0199\u0001\u0000\u0000\u0000\u019b\u019c\u0001\u0000\u0000\u0000"+ + "\u019c;\u0001\u0000\u0000\u0000\u019d\u019b\u0001\u0000\u0000\u0000\u019e"+ + "\u019f\u0007\u0003\u0000\u0000\u019f=\u0001\u0000\u0000\u0000\u01a0\u01a4"+ + "\u0005U\u0000\u0000\u01a1\u01a2\u0004\u001f\n\u0000\u01a2\u01a4\u0003"+ + "B!\u0000\u01a3\u01a0\u0001\u0000\u0000\u0000\u01a3\u01a1\u0001\u0000\u0000"+ + "\u0000\u01a4?\u0001\u0000\u0000\u0000\u01a5\u01d0\u00052\u0000\u0000\u01a6"+ + "\u01a7\u0003d2\u0000\u01a7\u01a8\u0005H\u0000\u0000\u01a8\u01d0\u0001"+ + "\u0000\u0000\u0000\u01a9\u01d0\u0003b1\u0000\u01aa\u01d0\u0003d2\u0000"+ + "\u01ab\u01d0\u0003^/\u0000\u01ac\u01d0\u0003B!\u0000\u01ad\u01d0\u0003"+ + "f3\u0000\u01ae\u01af\u0005F\u0000\u0000\u01af\u01b4\u0003`0\u0000\u01b0"+ + "\u01b1\u0005\'\u0000\u0000\u01b1\u01b3\u0003`0\u0000\u01b2\u01b0\u0001"+ + "\u0000\u0000\u0000\u01b3\u01b6\u0001\u0000\u0000\u0000\u01b4\u01b2\u0001"+ + "\u0000\u0000\u0000\u01b4\u01b5\u0001\u0000\u0000\u0000\u01b5\u01b7\u0001"+ + "\u0000\u0000\u0000\u01b6\u01b4\u0001\u0000\u0000\u0000\u01b7\u01b8\u0005"+ + "G\u0000\u0000\u01b8\u01d0\u0001\u0000\u0000\u0000\u01b9\u01ba\u0005F\u0000"+ + "\u0000\u01ba\u01bf\u0003^/\u0000\u01bb\u01bc\u0005\'\u0000\u0000\u01bc"+ + "\u01be\u0003^/\u0000\u01bd\u01bb\u0001\u0000\u0000\u0000\u01be\u01c1\u0001"+ + "\u0000\u0000\u0000\u01bf\u01bd\u0001\u0000\u0000\u0000\u01bf\u01c0\u0001"+ + "\u0000\u0000\u0000\u01c0\u01c2\u0001\u0000\u0000\u0000\u01c1\u01bf\u0001"+ + "\u0000\u0000\u0000\u01c2\u01c3\u0005G\u0000\u0000\u01c3\u01d0\u0001\u0000"+ + "\u0000\u0000\u01c4\u01c5\u0005F\u0000\u0000\u01c5\u01ca\u0003f3\u0000"+ + "\u01c6\u01c7\u0005\'\u0000\u0000\u01c7\u01c9\u0003f3\u0000\u01c8\u01c6"+ + "\u0001\u0000\u0000\u0000\u01c9\u01cc\u0001\u0000\u0000\u0000\u01ca\u01c8"+ + "\u0001\u0000\u0000\u0000\u01ca\u01cb\u0001\u0000\u0000\u0000\u01cb\u01cd"+ + "\u0001\u0000\u0000\u0000\u01cc\u01ca\u0001\u0000\u0000\u0000\u01cd\u01ce"+ + "\u0005G\u0000\u0000\u01ce\u01d0\u0001\u0000\u0000\u0000\u01cf\u01a5\u0001"+ + "\u0000\u0000\u0000\u01cf\u01a6\u0001\u0000\u0000\u0000\u01cf\u01a9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01aa\u0001\u0000\u0000\u0000\u01cf\u01ab\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ac\u0001\u0000\u0000\u0000\u01cf\u01ad\u0001"+ + "\u0000\u0000\u0000\u01cf\u01ae\u0001\u0000\u0000\u0000\u01cf\u01b9\u0001"+ + "\u0000\u0000\u0000\u01cf\u01c4\u0001\u0000\u0000\u0000\u01d0A\u0001\u0000"+ + "\u0000\u0000\u01d1\u01d4\u00055\u0000\u0000\u01d2\u01d4\u0005E\u0000\u0000"+ + "\u01d3\u01d1\u0001\u0000\u0000\u0000\u01d3\u01d2\u0001\u0000\u0000\u0000"+ + "\u01d4C\u0001\u0000\u0000\u0000\u01d5\u01d9\u0003<\u001e\u0000\u01d6\u01d7"+ + "\u0004\"\u000b\u0000\u01d7\u01d9\u0003B!\u0000\u01d8\u01d5\u0001\u0000"+ + "\u0000\u0000\u01d8\u01d6\u0001\u0000\u0000\u0000\u01d9E\u0001\u0000\u0000"+ + "\u0000\u01da\u01db\u0005\t\u0000\u0000\u01db\u01dc\u0005\u001f\u0000\u0000"+ + "\u01dcG\u0001\u0000\u0000\u0000\u01dd\u01de\u0005\u000e\u0000\u0000\u01de"+ + "\u01e3\u0003J%\u0000\u01df\u01e0\u0005\'\u0000\u0000\u01e0\u01e2\u0003"+ + "J%\u0000\u01e1\u01df\u0001\u0000\u0000\u0000\u01e2\u01e5\u0001\u0000\u0000"+ + "\u0000\u01e3\u01e1\u0001\u0000\u0000\u0000\u01e3\u01e4\u0001\u0000\u0000"+ + "\u0000\u01e4I\u0001\u0000\u0000\u0000\u01e5\u01e3\u0001\u0000\u0000\u0000"+ + "\u01e6\u01e8\u0003\n\u0005\u0000\u01e7\u01e9\u0007\u0004\u0000\u0000\u01e8"+ + "\u01e7\u0001\u0000\u0000\u0000\u01e8\u01e9\u0001\u0000\u0000\u0000\u01e9"+ + "\u01ec\u0001\u0000\u0000\u0000\u01ea\u01eb\u00053\u0000\u0000\u01eb\u01ed"+ + "\u0007\u0005\u0000\u0000\u01ec\u01ea\u0001\u0000\u0000\u0000\u01ec\u01ed"+ + "\u0001\u0000\u0000\u0000\u01edK\u0001\u0000\u0000\u0000\u01ee\u01ef\u0005"+ + "\b\u0000\u0000\u01ef\u01f0\u0003:\u001d\u0000\u01f0M\u0001\u0000\u0000"+ + "\u0000\u01f1\u01f2\u0005\u0002\u0000\u0000\u01f2\u01f3\u0003:\u001d\u0000"+ + "\u01f3O\u0001\u0000\u0000\u0000\u01f4\u01f5\u0005\u000b\u0000\u0000\u01f5"+ + "\u01fa\u0003R)\u0000\u01f6\u01f7\u0005\'\u0000\u0000\u01f7\u01f9\u0003"+ + "R)\u0000\u01f8\u01f6\u0001\u0000\u0000\u0000\u01f9\u01fc\u0001\u0000\u0000"+ + "\u0000\u01fa\u01f8\u0001\u0000\u0000\u0000\u01fa\u01fb\u0001\u0000\u0000"+ + "\u0000\u01fbQ\u0001\u0000\u0000\u0000\u01fc\u01fa\u0001\u0000\u0000\u0000"+ + "\u01fd\u01fe\u00038\u001c\u0000\u01fe\u01ff\u0005Y\u0000\u0000\u01ff\u0200"+ + "\u00038\u001c\u0000\u0200S\u0001\u0000\u0000\u0000\u0201\u0202\u0005\u0001"+ + "\u0000\u0000\u0202\u0203\u0003\u0014\n\u0000\u0203\u0205\u0003f3\u0000"+ + "\u0204\u0206\u0003Z-\u0000\u0205\u0204\u0001\u0000\u0000\u0000\u0205\u0206"+ + "\u0001\u0000\u0000\u0000\u0206U\u0001\u0000\u0000\u0000\u0207\u0208\u0005"+ + "\u0007\u0000\u0000\u0208\u0209\u0003\u0014\n\u0000\u0209\u020a\u0003f"+ + "3\u0000\u020aW\u0001\u0000\u0000\u0000\u020b\u020c\u0005\n\u0000\u0000"+ + "\u020c\u020d\u00036\u001b\u0000\u020dY\u0001\u0000\u0000\u0000\u020e\u0213"+ + "\u0003\\.\u0000\u020f\u0210\u0005\'\u0000\u0000\u0210\u0212\u0003\\.\u0000"+ + "\u0211\u020f\u0001\u0000\u0000\u0000\u0212\u0215\u0001\u0000\u0000\u0000"+ + "\u0213\u0211\u0001\u0000\u0000\u0000\u0213\u0214\u0001\u0000\u0000\u0000"+ + "\u0214[\u0001\u0000\u0000\u0000\u0215\u0213\u0001\u0000\u0000\u0000\u0216"+ + "\u0217\u0003<\u001e\u0000\u0217\u0218\u0005$\u0000\u0000\u0218\u0219\u0003"+ + "@ \u0000\u0219]\u0001\u0000\u0000\u0000\u021a\u021b\u0007\u0006\u0000"+ + "\u0000\u021b_\u0001\u0000\u0000\u0000\u021c\u021f\u0003b1\u0000\u021d"+ + "\u021f\u0003d2\u0000\u021e\u021c\u0001\u0000\u0000\u0000\u021e\u021d\u0001"+ + "\u0000\u0000\u0000\u021fa\u0001\u0000\u0000\u0000\u0220\u0222\u0007\u0000"+ + "\u0000\u0000\u0221\u0220\u0001\u0000\u0000\u0000\u0221\u0222\u0001\u0000"+ + "\u0000\u0000\u0222\u0223\u0001\u0000\u0000\u0000\u0223\u0224\u0005 \u0000"+ + "\u0000\u0224c\u0001\u0000\u0000\u0000\u0225\u0227\u0007\u0000\u0000\u0000"+ + "\u0226\u0225\u0001\u0000\u0000\u0000\u0226\u0227\u0001\u0000\u0000\u0000"+ + "\u0227\u0228\u0001\u0000\u0000\u0000\u0228\u0229\u0005\u001f\u0000\u0000"+ + "\u0229e\u0001\u0000\u0000\u0000\u022a\u022b\u0005\u001e\u0000\u0000\u022b"+ + "g\u0001\u0000\u0000\u0000\u022c\u022d\u0007\u0007\u0000\u0000\u022di\u0001"+ + "\u0000\u0000\u0000\u022e\u022f\u0005\u0005\u0000\u0000\u022f\u0230\u0003"+ + "l6\u0000\u0230k\u0001\u0000\u0000\u0000\u0231\u0232\u0005F\u0000\u0000"+ + "\u0232\u0233\u0003\u0002\u0001\u0000\u0233\u0234\u0005G\u0000\u0000\u0234"+ + "m\u0001\u0000\u0000\u0000\u0235\u0236\u0005\r\u0000\u0000\u0236\u0237"+ + "\u0005i\u0000\u0000\u0237o\u0001\u0000\u0000\u0000\u0238\u0239\u0005\u0003"+ + "\u0000\u0000\u0239\u023c\u0005_\u0000\u0000\u023a\u023b\u0005]\u0000\u0000"+ + "\u023b\u023d\u00038\u001c\u0000\u023c\u023a\u0001\u0000\u0000\u0000\u023c"+ + "\u023d\u0001\u0000\u0000\u0000\u023d\u0247\u0001\u0000\u0000\u0000\u023e"+ + "\u023f\u0005^\u0000\u0000\u023f\u0244\u0003r9\u0000\u0240\u0241\u0005"+ + "\'\u0000\u0000\u0241\u0243\u0003r9\u0000\u0242\u0240\u0001\u0000\u0000"+ + "\u0000\u0243\u0246\u0001\u0000\u0000\u0000\u0244\u0242\u0001\u0000\u0000"+ + "\u0000\u0244\u0245\u0001\u0000\u0000\u0000\u0245\u0248\u0001\u0000\u0000"+ + "\u0000\u0246\u0244\u0001\u0000\u0000\u0000\u0247\u023e\u0001\u0000\u0000"+ + "\u0000\u0247\u0248\u0001\u0000\u0000\u0000\u0248q\u0001\u0000\u0000\u0000"+ + "\u0249\u024a\u00038\u001c\u0000\u024a\u024b\u0005$\u0000\u0000\u024b\u024d"+ + "\u0001\u0000\u0000\u0000\u024c\u0249\u0001\u0000\u0000\u0000\u024c\u024d"+ + "\u0001\u0000\u0000\u0000\u024d\u024e\u0001\u0000\u0000\u0000\u024e\u024f"+ + "\u00038\u001c\u0000\u024fs\u0001\u0000\u0000\u0000\u0250\u0251\u0005\u0012"+ + "\u0000\u0000\u0251\u0252\u0003$\u0012\u0000\u0252\u0253\u0005]\u0000\u0000"+ + "\u0253\u0254\u0003:\u001d\u0000\u0254u\u0001\u0000\u0000\u0000\u0255\u0256"+ + "\u0005\u0011\u0000\u0000\u0256\u0259\u00032\u0019\u0000\u0257\u0258\u0005"+ + "!\u0000\u0000\u0258\u025a\u0003\u001e\u000f\u0000\u0259\u0257\u0001\u0000"+ + "\u0000\u0000\u0259\u025a\u0001\u0000\u0000\u0000\u025aw\u0001\u0000\u0000"+ + "\u0000\u025b\u025d\u0007\b\u0000\u0000\u025c\u025b\u0001\u0000\u0000\u0000"+ + "\u025c\u025d\u0001\u0000\u0000\u0000\u025d\u025e\u0001\u0000\u0000\u0000"+ + "\u025e\u025f\u0005\u0014\u0000\u0000\u025f\u0260\u0003z=\u0000\u0260\u0261"+ + "\u0003|>\u0000\u0261y\u0001\u0000\u0000\u0000\u0262\u0265\u0003<\u001e"+ + "\u0000\u0263\u0264\u0005Y\u0000\u0000\u0264\u0266\u0003<\u001e\u0000\u0265"+ + "\u0263\u0001\u0000\u0000\u0000\u0265\u0266\u0001\u0000\u0000\u0000\u0266"+ + "{\u0001\u0000\u0000\u0000\u0267\u0268\u0005]\u0000\u0000\u0268\u026d\u0003"+ + "~?\u0000\u0269\u026a\u0005\'\u0000\u0000\u026a\u026c\u0003~?\u0000\u026b"+ + "\u0269\u0001\u0000\u0000\u0000\u026c\u026f\u0001\u0000\u0000\u0000\u026d"+ + "\u026b\u0001\u0000\u0000\u0000\u026d\u026e\u0001\u0000\u0000\u0000\u026e"+ + "}\u0001\u0000\u0000\u0000\u026f\u026d\u0001\u0000\u0000\u0000\u0270\u0271"+ + "\u0003\u0010\b\u0000\u0271\u007f\u0001\u0000\u0000\u0000=\u008b\u0094"+ + "\u00a8\u00b4\u00bd\u00c5\u00ca\u00d2\u00d4\u00d9\u00e0\u00e5\u00ea\u00f4"+ + "\u00fa\u0102\u0104\u010f\u0116\u0121\u0124\u0134\u013a\u0144\u0148\u014d"+ + "\u015b\u0164\u0168\u016c\u0173\u0177\u017e\u0184\u018b\u0193\u019b\u01a3"+ + "\u01b4\u01bf\u01ca\u01cf\u01d3\u01d8\u01e3\u01e8\u01ec\u01fa\u0205\u0213"+ + "\u021e\u0221\u0226\u023c\u0244\u0247\u024c\u0259\u025c\u0265\u026d"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java index 6071219839bab..2ee0efe52dfbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseListener.java @@ -452,30 +452,6 @@ public class EsqlBaseParserBaseListener implements EsqlBaseParserListener { *

The default implementation does nothing.

*/ @Override public void exitMetadata(EsqlBaseParser.MetadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } - /** - * {@inheritDoc} - * - *

The default implementation does nothing.

- */ - @Override public void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java index afe7146923791..c5c1b0dfa7d7d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserBaseVisitor.java @@ -272,20 +272,6 @@ public class EsqlBaseParserBaseVisitor extends AbstractParseTreeVisitor im * {@link #visitChildren} on {@code ctx}.

*/ @Override public T visitMetadata(EsqlBaseParser.MetadataContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx) { return visitChildren(ctx); } - /** - * {@inheritDoc} - * - *

The default implementation returns the result of calling - * {@link #visitChildren} on {@code ctx}.

- */ - @Override public T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java index 0faca2541c9ad..f45184e920658 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserListener.java @@ -415,26 +415,6 @@ public interface EsqlBaseParserListener extends ParseTreeListener { * @param ctx the parse tree */ void exitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void enterMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - */ - void exitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Enter a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void enterDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); - /** - * Exit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - */ - void exitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Enter a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java index e91cd6670e971..30c5e0ce78092 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParserVisitor.java @@ -254,18 +254,6 @@ public interface EsqlBaseParserVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitMetadata(EsqlBaseParser.MetadataContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#metadataOption}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitMetadataOption(EsqlBaseParser.MetadataOptionContext ctx); - /** - * Visit a parse tree produced by {@link EsqlBaseParser#deprecated_metadata}. - * @param ctx the parse tree - * @return the visitor result - */ - T visitDeprecated_metadata(EsqlBaseParser.Deprecated_metadataContext ctx); /** * Visit a parse tree produced by {@link EsqlBaseParser#metricsCommand}. * @param ctx the parse tree diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 81d43bc68b79e..eb81446f9ddea 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; @@ -686,12 +687,20 @@ public Expression visitRegexBooleanExpression(EsqlBaseParser.RegexBooleanExpress RegexMatch result = switch (type) { case EsqlBaseParser.LIKE -> { try { - yield new WildcardLike(source, left, new WildcardPattern(pattern.fold().toString())); + yield new WildcardLike( + source, + left, + new WildcardPattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString()) + ); } catch (InvalidArgumentException e) { throw new ParsingException(source, "Invalid pattern for LIKE [{}]: [{}]", pattern, e.getMessage()); } } - case EsqlBaseParser.RLIKE -> new RLike(source, left, new RLikePattern(pattern.fold().toString())); + case EsqlBaseParser.RLIKE -> new RLike( + source, + left, + new RLikePattern(pattern.fold(FoldContext.small() /* TODO remove me */).toString()) + ); default -> throw new ParsingException("Invalid predicate type for [{}]", source.text()); }; return ctx.NOT() == null ? result : new Not(source, result); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java index 49d77bc36fb2e..4b7c0118acda3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java @@ -23,6 +23,7 @@ import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -34,7 +35,6 @@ import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; -import org.elasticsearch.xpack.esql.parser.EsqlBaseParser.MetadataOptionContext; import org.elasticsearch.xpack.esql.plan.TableIdentifier; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; @@ -71,7 +71,6 @@ import java.util.function.Function; import static java.util.Collections.emptyList; -import static org.elasticsearch.common.logging.HeaderWarning.addWarning; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; import static org.elasticsearch.xpack.esql.parser.ParserUtils.source; @@ -157,7 +156,7 @@ public PlanFactory visitEvalCommand(EsqlBaseParser.EvalCommandContext ctx) { public PlanFactory visitGrokCommand(EsqlBaseParser.GrokCommandContext ctx) { return p -> { Source source = source(ctx); - String pattern = visitString(ctx.string()).fold().toString(); + String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString(); Grok.Parser grokParser; try { grokParser = Grok.pattern(source, pattern); @@ -188,7 +187,7 @@ private void validateGrokPattern(Source source, Grok.Parser grokParser, String p @Override public PlanFactory visitDissectCommand(EsqlBaseParser.DissectCommandContext ctx) { return p -> { - String pattern = visitString(ctx.string()).fold().toString(); + String pattern = visitString(ctx.string()).fold(FoldContext.small() /* TODO remove me */).toString(); Map options = visitCommandOptions(ctx.commandOptions()); String appendSeparator = ""; for (Map.Entry item : options.entrySet()) { @@ -243,7 +242,7 @@ public Map visitCommandOptions(EsqlBaseParser.CommandOptionsCont } Map result = new HashMap<>(); for (EsqlBaseParser.CommandOptionContext option : ctx.commandOption()) { - result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold()); + result.put(visitIdentifier(option.identifier()), expression(option.constant()).fold(FoldContext.small() /* TODO remove me */)); } return result; } @@ -260,21 +259,7 @@ public LogicalPlan visitFromCommand(EsqlBaseParser.FromCommandContext ctx) { TableIdentifier table = new TableIdentifier(source, null, visitIndexPattern(ctx.indexPattern())); Map metadataMap = new LinkedHashMap<>(); if (ctx.metadata() != null) { - var deprecatedContext = ctx.metadata().deprecated_metadata(); - MetadataOptionContext metadataOptionContext = null; - if (deprecatedContext != null) { - var s = source(deprecatedContext).source(); - addWarning( - "Line {}:{}: Square brackets '[]' need to be removed in FROM METADATA declaration", - s.getLineNumber(), - s.getColumnNumber() - ); - metadataOptionContext = deprecatedContext.metadataOption(); - } else { - metadataOptionContext = ctx.metadata().metadataOption(); - - } - for (var c : metadataOptionContext.UNQUOTED_SOURCE()) { + for (var c : ctx.metadata().UNQUOTED_SOURCE()) { String id = c.getText(); Source src = source(c); if (MetadataAttribute.isSupported(id) == false // TODO: drop check below once METADATA_SCORE is no longer snapshot-only diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 6755a7fa30af9..9b81060349815 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -149,7 +150,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeNamedWriteable(policyName()); out.writeNamedWriteable(matchField()); if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) { - out.writeString(BytesRefs.toString(policyName().fold())); // old policy name + out.writeString(BytesRefs.toString(policyName().fold(FoldContext.small() /* TODO remove me */))); // old policy name } policy().writeTo(out); if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 57ba1c8016feb..072bae21da2a3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; @@ -47,9 +48,11 @@ public abstract class AbstractPhysicalOperationProviders implements PhysicalOperationProviders { private final AggregateMapper aggregateMapper = new AggregateMapper(); + private final FoldContext foldContext; private final AnalysisRegistry analysisRegistry; - AbstractPhysicalOperationProviders(AnalysisRegistry analysisRegistry) { + AbstractPhysicalOperationProviders(FoldContext foldContext, AnalysisRegistry analysisRegistry) { + this.foldContext = foldContext; this.analysisRegistry = analysisRegistry; } @@ -251,6 +254,7 @@ public static List intermediateAttributes(List aggregates, AggregatorMode mode, Layout layout, @@ -311,7 +315,11 @@ else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { // apply the filter only in the initial phase - as the rest of the data is already filtered if (aggregateFunction.hasFilter() && mode.isInputPartial() == false) { - EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator(aggregateFunction.filter(), layout); + EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator( + foldContext, + aggregateFunction.filter(), + layout + ); aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory); } consumer.accept(new AggFunctionSupplierContext(aggSupplier, mode)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index b1fe0e7a7cf54..112f101ad842b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -33,7 +33,6 @@ import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NestedLookup; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -47,6 +46,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; @@ -94,8 +94,8 @@ public interface ShardContext extends org.elasticsearch.compute.lucene.ShardCont private final List shardContexts; - public EsPhysicalOperationProviders(List shardContexts, AnalysisRegistry analysisRegistry) { - super(analysisRegistry); + public EsPhysicalOperationProviders(FoldContext foldContext, List shardContexts, AnalysisRegistry analysisRegistry) { + super(foldContext, analysisRegistry); this.shardContexts = shardContexts; } @@ -161,7 +161,7 @@ public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, List sorts = esQueryExec.sorts(); assert esQueryExec.estimatedRowSize() != null : "estimated row size not initialized"; int rowEstimatedSize = esQueryExec.estimatedRowSize(); - int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold() : NO_LIMIT; + int limit = esQueryExec.limit() != null ? (Integer) esQueryExec.limit().fold(context.foldCtx()) : NO_LIMIT; boolean scoring = esQueryExec.attrs() .stream() .anyMatch(a -> a instanceof MetadataAttribute && a.name().equals(MetadataAttribute.SCORE)); @@ -217,7 +217,7 @@ public LuceneCountOperator.Factory countSource(LocalExecutionPlannerContext cont querySupplier(queryBuilder), context.queryPragmas().dataPartitioning(), context.queryPragmas().taskConcurrency(), - limit == null ? NO_LIMIT : (Integer) limit.fold() + limit == null ? NO_LIMIT : (Integer) limit.fold(context.foldCtx()) ); } @@ -338,16 +338,7 @@ public MappedFieldType.FieldExtractPreference fieldExtractPreference() { @Override public SearchLookup lookup() { - boolean syntheticSource = SourceFieldMapper.isSynthetic(indexSettings()); - var searchLookup = ctx.lookup(); - if (syntheticSource) { - // in the context of scripts and when synthetic source is used the search lookup can't always be reused between - // users of SearchLookup. This is only an issue when scripts fallback to _source, but since we can't always - // accurately determine whether a script uses _source, we should do this for all script usages. - // This lookup() method is only invoked for scripts / runtime fields, so it is ok to do here. - searchLookup = searchLookup.swapSourceProvider(ctx.createSourceProvider()); - } - return searchLookup; + return ctx.lookup(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java index a1765977ee9c2..c185bd5729879 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsqlExpressionTranslators.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.TranslationAware; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; @@ -144,7 +145,7 @@ public static void checkInsensitiveComparison(InsensitiveEquals bc) { static Query translate(InsensitiveEquals bc) { TypedAttribute attribute = checkIsPushableAttribute(bc.left()); Source source = bc.source(); - BytesRef value = BytesRefs.toBytesRef(valueOf(bc.right())); + BytesRef value = BytesRefs.toBytesRef(valueOf(FoldContext.small() /* TODO remove me */, bc.right())); String name = pushableAttributeName(attribute); return new TermQuery(source, name, value.utf8ToString(), true); } @@ -188,7 +189,7 @@ static Query translate(BinaryComparison bc, TranslatorHandler handler) { TypedAttribute attribute = checkIsPushableAttribute(bc.left()); Source source = bc.source(); String name = handler.nameOf(attribute); - Object result = bc.right().fold(); + Object result = bc.right().fold(FoldContext.small() /* TODO remove me */); Object value = result; String format = null; boolean isDateLiteralComparison = false; @@ -269,7 +270,7 @@ private static Query translateOutOfRangeComparisons(BinaryComparison bc) { return null; } Source source = bc.source(); - Object value = valueOf(bc.right()); + Object value = valueOf(FoldContext.small() /* TODO remove me */, bc.right()); // Comparisons with multi-values always return null in ESQL. if (value instanceof List) { @@ -369,7 +370,7 @@ public static Query doTranslate(ScalarFunction f, TranslatorHandler handler) { if (f instanceof CIDRMatch cm) { if (cm.ipField() instanceof FieldAttribute fa && Expressions.foldable(cm.matches())) { String targetFieldName = handler.nameOf(fa.exactAttribute()); - Set set = new LinkedHashSet<>(Expressions.fold(cm.matches())); + Set set = new LinkedHashSet<>(Expressions.fold(FoldContext.small() /* TODO remove me */, cm.matches())); Query query = new TermsQuery(f.source(), targetFieldName, set); // CIDR_MATCH applies only to single values. @@ -420,7 +421,7 @@ static Query translate( String name = handler.nameOf(attribute); try { - Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(constantExpression); + Geometry shape = SpatialRelatesUtils.makeGeometryFromLiteral(FoldContext.small() /* TODO remove me */, constantExpression); return new SpatialRelatesQuery(bc.source(), name, bc.queryRelation(), shape, attribute.dataType()); } catch (IllegalArgumentException e) { throw new QlIllegalArgumentException(e.getMessage(), e); @@ -461,7 +462,7 @@ private static Query translate(In in, TranslatorHandler handler) { queries.add(query); } } else { - terms.add(valueOf(rhs)); + terms.add(valueOf(FoldContext.small() /* TODO remove me */, rhs)); } } } @@ -487,8 +488,8 @@ public static Query doTranslate(Range r, TranslatorHandler handler) { } private static RangeQuery translate(Range r, TranslatorHandler handler) { - Object lower = valueOf(r.lower()); - Object upper = valueOf(r.upper()); + Object lower = valueOf(FoldContext.small() /* TODO remove me */, r.lower()); + Object upper = valueOf(FoldContext.small() /* TODO remove me */, r.upper()); String format = null; DataType dataType = r.value().dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index af38551c1ad06..ecd0284c7cb57 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -22,7 +22,6 @@ import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.EvalOperatorFactory; -import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.compute.operator.FilterOperator.FilterOperatorFactory; import org.elasticsearch.compute.operator.LocalSourceOperator; import org.elasticsearch.compute.operator.LocalSourceOperator.LocalSourceFactory; @@ -55,6 +54,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -161,13 +161,14 @@ public LocalExecutionPlanner( /** * turn the given plan into a list of drivers to execute */ - public LocalExecutionPlan plan(PhysicalPlan localPhysicalPlan) { + public LocalExecutionPlan plan(FoldContext foldCtx, PhysicalPlan localPhysicalPlan) { var context = new LocalExecutionPlannerContext( new ArrayList<>(), new Holder<>(DriverParallelism.SINGLE), configuration.pragmas(), bigArrays, blockFactory, + foldCtx, settings ); @@ -397,7 +398,7 @@ private PhysicalOperation planEval(EvalExec eval, LocalExecutionPlannerContext c PhysicalOperation source = plan(eval.child(), context); for (Alias field : eval.fields()) { - var evaluatorSupplier = EvalMapper.toEvaluator(field.child(), source.layout); + var evaluatorSupplier = EvalMapper.toEvaluator(context.foldCtx(), field.child(), source.layout); Layout.Builder layout = source.layout.builder(); layout.append(field.toAttribute()); source = source.with(new EvalOperatorFactory(evaluatorSupplier), layout.build()); @@ -418,7 +419,7 @@ private PhysicalOperation planDissect(DissectExec dissect, LocalExecutionPlanner source = source.with( new StringExtractOperator.StringExtractOperatorFactory( patternNames, - EvalMapper.toEvaluator(expr, layout), + EvalMapper.toEvaluator(context.foldCtx(), expr, layout), () -> (input) -> dissect.parser().parser().parse(input) ), layout @@ -450,7 +451,7 @@ private PhysicalOperation planGrok(GrokExec grok, LocalExecutionPlannerContext c source = source.with( new ColumnExtractOperator.Factory( types, - EvalMapper.toEvaluator(grok.inputExpression(), layout), + EvalMapper.toEvaluator(context.foldCtx(), grok.inputExpression(), layout), () -> new GrokEvaluatorExtracter(grok.pattern().grok(), grok.pattern().pattern(), fieldToPos, fieldToType) ), layout @@ -599,10 +600,6 @@ private PhysicalOperation planLookupJoin(LookupJoinExec join, LocalExecutionPlan ); } - private ExpressionEvaluator.Factory toEvaluator(Expression exp, Layout layout) { - return EvalMapper.toEvaluator(exp, layout); - } - private PhysicalOperation planLocal(LocalSourceExec localSourceExec, LocalExecutionPlannerContext context) { Layout.Builder layout = new Layout.Builder(); layout.append(localSourceExec.output()); @@ -657,12 +654,15 @@ private PhysicalOperation planProject(ProjectExec project, LocalExecutionPlanner private PhysicalOperation planFilter(FilterExec filter, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(filter.child(), context); // TODO: should this be extracted into a separate eval block? - return source.with(new FilterOperatorFactory(toEvaluator(filter.condition(), source.layout)), source.layout); + return source.with( + new FilterOperatorFactory(EvalMapper.toEvaluator(context.foldCtx(), filter.condition(), source.layout)), + source.layout + ); } private PhysicalOperation planLimit(LimitExec limit, LocalExecutionPlannerContext context) { PhysicalOperation source = plan(limit.child(), context); - return source.with(new Factory((Integer) limit.limit().fold()), source.layout); + return source.with(new Factory((Integer) limit.limit().fold(context.foldCtx)), source.layout); } private PhysicalOperation planMvExpand(MvExpandExec mvExpandExec, LocalExecutionPlannerContext context) { @@ -783,6 +783,7 @@ public record LocalExecutionPlannerContext( QueryPragmas queryPragmas, BigArrays bigArrays, BlockFactory blockFactory, + FoldContext foldCtx, Settings settings ) { void addDriverFactory(DriverFactory driverFactory) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 5325145a77ade..2f4368155069f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -159,13 +160,18 @@ private static , E extends T> void forEachUpWithChildren( } } - public static PhysicalPlan localPlan(List searchContexts, Configuration configuration, PhysicalPlan plan) { - return localPlan(configuration, plan, SearchContextStats.from(searchContexts)); + public static PhysicalPlan localPlan( + List searchContexts, + Configuration configuration, + FoldContext foldCtx, + PhysicalPlan plan + ) { + return localPlan(configuration, foldCtx, plan, SearchContextStats.from(searchContexts)); } - public static PhysicalPlan localPlan(Configuration configuration, PhysicalPlan plan, SearchStats searchStats) { - final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats)); - var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats)); + public static PhysicalPlan localPlan(Configuration configuration, FoldContext foldCtx, PhysicalPlan plan, SearchStats searchStats) { + final var logicalOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalOptimizer = new LocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats)); return localPlan(plan, logicalOptimizer, physicalOptimizer); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java index 334875927eb96..4dea8a50b5c17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TypeConverter.java @@ -14,6 +14,9 @@ import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; class TypeConverter { @@ -33,19 +36,26 @@ public static TypeConverter fromConvertFunction(AbstractConvertFunction convertF BigArrays.NON_RECYCLING_INSTANCE ) ); - return new TypeConverter( - convertFunction.functionName(), - convertFunction.toEvaluator(e -> driverContext -> new ExpressionEvaluator() { - @Override - public org.elasticsearch.compute.data.Block eval(Page page) { - // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) - return page.getBlock(0); - } - - @Override - public void close() {} - }).get(driverContext1) - ); + return new TypeConverter(convertFunction.functionName(), convertFunction.toEvaluator(new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return driverContext -> new ExpressionEvaluator() { + @Override + public org.elasticsearch.compute.data.Block eval(Page page) { + // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) + return page.getBlock(0); + } + + @Override + public void close() {} + }; + } + + @Override + public FoldContext foldCtx() { + throw new IllegalStateException("not folding"); + } + }).get(driverContext1)); } public Block convert(Block block) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java index e881eabb38c43..b8f539ea307c9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/mapper/MapperUtils.java @@ -11,6 +11,7 @@ import org.elasticsearch.compute.aggregation.AggregatorMode; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -90,7 +91,7 @@ static PhysicalPlan mapUnary(UnaryPlan p, PhysicalPlan child) { enrich.mode(), enrich.policy().getType(), enrich.matchField(), - BytesRefs.toString(enrich.policyName().fold()), + BytesRefs.toString(enrich.policyName().fold(FoldContext.small() /* TODO remove me */)), enrich.policy().getMatchField(), enrich.concreteIndices(), enrich.enrichFields() diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java index 7223e6988bb19..a38236fe60954 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ComputeService.java @@ -61,6 +61,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryAction; import org.elasticsearch.xpack.esql.action.EsqlSearchShardsAction; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; import org.elasticsearch.xpack.esql.plan.physical.ExchangeSinkExec; @@ -140,6 +141,7 @@ public void execute( CancellableTask rootTask, PhysicalPlan physicalPlan, Configuration configuration, + FoldContext foldContext, EsqlExecutionInfo execInfo, ActionListener listener ) { @@ -174,6 +176,7 @@ public void execute( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, List.of(), configuration, + foldContext, null, null ); @@ -226,6 +229,7 @@ public void execute( RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY, List.of(), configuration, + foldContext, exchangeSource, null ), @@ -460,16 +464,16 @@ public SourceProvider createSourceProvider() { context.exchangeSink(), enrichLookupService, lookupFromIndexService, - new EsPhysicalOperationProviders(contexts, searchService.getIndicesService().getAnalysis()) + new EsPhysicalOperationProviders(context.foldCtx(), contexts, searchService.getIndicesService().getAnalysis()) ); LOGGER.debug("Received physical plan:\n{}", plan); - plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, plan); + plan = PlannerUtils.localPlan(context.searchExecutionContexts(), context.configuration, context.foldCtx(), plan); // the planner will also set the driver parallelism in LocalExecutionPlanner.LocalExecutionPlan (used down below) // it's doing this in the planning of EsQueryExec (the source of the data) // see also EsPhysicalOperationProviders.sourcePhysicalOperation - LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(plan); + LocalExecutionPlanner.LocalExecutionPlan localExecutionPlan = planner.plan(context.foldCtx(), plan); if (LOGGER.isDebugEnabled()) { LOGGER.debug("Local execution plan:\n{}", localExecutionPlan.describe()); } @@ -715,7 +719,15 @@ public void onFailure(Exception e) { }; acquireSearchContexts(clusterAlias, shardIds, configuration, request.aliasFilters(), ActionListener.wrap(searchContexts -> { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH, ESQL_WORKER_THREAD_POOL_NAME); - var computeContext = new ComputeContext(sessionId, clusterAlias, searchContexts, configuration, null, exchangeSink); + var computeContext = new ComputeContext( + sessionId, + clusterAlias, + searchContexts, + configuration, + configuration.newFoldContext(), + null, + exchangeSink + ); runCompute(parentTask, computeContext, request.plan(), batchListener); }, batchListener::onFailure)); } @@ -766,6 +778,7 @@ private void runComputeOnDataNode( request.clusterAlias(), List.of(), request.configuration(), + new FoldContext(request.pragmas().foldLimit().getBytes()), exchangeSource, externalSink ), @@ -901,7 +914,15 @@ void runComputeOnRemoteCluster( exchangeSink.addCompletionListener(computeListener.acquireAvoid()); runCompute( parentTask, - new ComputeContext(localSessionId, clusterAlias, List.of(), configuration, exchangeSource, exchangeSink), + new ComputeContext( + localSessionId, + clusterAlias, + List.of(), + configuration, + configuration.newFoldContext(), + exchangeSource, + exchangeSink + ), coordinatorPlan, computeListener.acquireCompute(clusterAlias) ); @@ -925,6 +946,7 @@ record ComputeContext( String clusterAlias, List searchContexts, Configuration configuration, + FoldContext foldCtx, ExchangeSourceHandler exchangeSource, ExchangeSinkHandler exchangeSink ) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java index 58e80e569ee5e..2443c3f2cda62 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/QueryPragmas.java @@ -12,12 +12,14 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.compute.lucene.DataPartitioning; import org.elasticsearch.compute.operator.Driver; import org.elasticsearch.compute.operator.DriverStatus; import org.elasticsearch.core.TimeValue; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.esql.core.expression.Expression; import java.io.IOException; import java.util.Objects; @@ -57,6 +59,8 @@ public final class QueryPragmas implements Writeable { public static final Setting NODE_LEVEL_REDUCTION = Setting.boolSetting("node_level_reduction", true); + public static final Setting FOLD_LIMIT = Setting.memorySizeSetting("fold_limit", "5%"); + public static final QueryPragmas EMPTY = new QueryPragmas(Settings.EMPTY); private final Settings settings; @@ -134,6 +138,17 @@ public boolean nodeLevelReduction() { return NODE_LEVEL_REDUCTION.get(settings); } + /** + * The maximum amount of memory we can use for {@link Expression#fold} during planing. This + * defaults to 5% of memory available on the current node. If this method is called on the + * coordinating node, this is 5% of the coordinating node's memory. If it's called on a data + * node, it's 5% of the data node. That's an exciting inconsistency. But it's + * important. Bigger nodes have more space to do folding. + */ + public ByteSizeValue foldLimit() { + return FOLD_LIMIT.get(settings); + } + public boolean isEmpty() { return settings.isEmpty(); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java index 8dee3478b3b64..61ac67674b252 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/ReinitializingSourceProvider.java @@ -28,10 +28,6 @@ final class ReinitializingSourceProvider implements SourceProvider { private PerThreadSourceProvider perThreadProvider; private final Supplier sourceProviderFactory; - // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: - // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) - private int lastSeenDocId; - ReinitializingSourceProvider(Supplier sourceProviderFactory) { this.sourceProviderFactory = sourceProviderFactory; } @@ -40,15 +36,25 @@ final class ReinitializingSourceProvider implements SourceProvider { public Source getSource(LeafReaderContext ctx, int doc) throws IOException { var currentThread = Thread.currentThread(); PerThreadSourceProvider provider = perThreadProvider; - if (provider == null || provider.creatingThread != currentThread || doc < lastSeenDocId) { + if (provider == null || provider.creatingThread != currentThread || doc < provider.lastSeenDocId) { provider = new PerThreadSourceProvider(sourceProviderFactory.get(), currentThread); this.perThreadProvider = provider; } - lastSeenDocId = doc; + provider.lastSeenDocId = doc; return provider.source.getSource(ctx, doc); } - private record PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + private static final class PerThreadSourceProvider { + final SourceProvider source; + final Thread creatingThread; + // Keeping track of last seen doc and if current doc is before last seen doc then source provider is initialized: + // (when source mode is synthetic then _source is read from doc values and doc values don't support going backwards) + int lastSeenDocId; + + private PerThreadSourceProvider(SourceProvider source, Thread creatingThread) { + this.source = source; + this.creatingThread = creatingThread; + } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java index b44e249e38006..84173eeecc060 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlQueryAction.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryResponse; import org.elasticsearch.xpack.esql.action.EsqlQueryTask; import org.elasticsearch.xpack.esql.core.async.AsyncTaskManagementService; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichLookupService; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.enrich.LookupFromIndexService; @@ -189,11 +190,13 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener computeService.execute( sessionId, (CancellableTask) task, plan, configuration, + foldCtx, executionInfo, resultListener ); @@ -201,6 +204,7 @@ private void innerExecute(Task task, EsqlQueryRequest request, ActionListener new EnrichPolicyResolver.UnresolvedPolicy((String) e.policyName().fold(), e.mode())) + .map( + e -> new EnrichPolicyResolver.UnresolvedPolicy( + (String) e.policyName().fold(FoldContext.small() /* TODO remove me*/), + e.mode() + ) + ) .collect(Collectors.toSet()); final List indices = preAnalysis.indices; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 0847f71b1fb01..95ee6ab337bd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.Converter; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -239,9 +240,9 @@ public static Converter converterFor(DataType from, DataType to) { return null; } - public static TemporalAmount foldToTemporalAmount(Expression field, String sourceText, DataType expectedType) { + public static TemporalAmount foldToTemporalAmount(FoldContext ctx, Expression field, String sourceText, DataType expectedType) { if (field.foldable()) { - Object v = field.fold(); + Object v = field.fold(ctx); if (v instanceof BytesRef b) { try { return EsqlDataTypeConverter.parseTemporalAmount(b.utf8ToString(), expectedType); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 58576eac42dfd..6cbf910906aaf 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -55,6 +55,7 @@ import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; @@ -485,15 +486,16 @@ private static CsvTestsDataLoader.MultiIndexTestDataset testDatasets(LogicalPlan return new CsvTestsDataLoader.MultiIndexTestDataset(indexName, datasets); } - private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.MultiIndexTestDataset datasets) - throws Exception { - var indexResolution = loadIndexResolution(datasets); + private static TestPhysicalOperationProviders testOperationProviders( + FoldContext foldCtx, + CsvTestsDataLoader.MultiIndexTestDataset datasets + ) throws Exception { var indexPages = new ArrayList(); for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { var testData = loadPageFromCsv(CsvTests.class.getResource("/data/" + dataset.dataFileName()), dataset.typeMapping()); indexPages.add(new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2())); } - return TestPhysicalOperationProviders.create(indexPages); + return TestPhysicalOperationProviders.create(foldCtx, indexPages); } private ActualResults executePlan(BigArrays bigArrays) throws Exception { @@ -501,6 +503,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { var testDatasets = testDatasets(parsed); LogicalPlan analyzed = analyzedPlan(parsed, testDatasets); + FoldContext foldCtx = FoldContext.small(); EsqlSession session = new EsqlSession( getTestName(), configuration, @@ -508,21 +511,21 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { null, null, functionRegistry, - new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration)), + new LogicalPlanOptimizer(new LogicalOptimizerContext(configuration, foldCtx)), mapper, TEST_VERIFIER, new PlanningMetrics(), null, EsqlTestUtils.MOCK_QUERY_BUILDER_RESOLVER ); - TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(testDatasets); + TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets); PlainActionFuture listener = new PlainActionFuture<>(); session.executeOptimizedPlan( new EsqlQueryRequest(), new EsqlExecutionInfo(randomBoolean()), - planRunner(bigArrays, physicalOperationProviders), + planRunner(bigArrays, foldCtx, physicalOperationProviders), session.optimizedPlan(analyzed), listener.delegateFailureAndWrap( // Wrap so we can capture the warnings in the calling thread @@ -582,12 +585,13 @@ private void assertWarnings(List warnings) { testCase.assertWarnings(false).assertWarnings(normalized); } - PlanRunner planRunner(BigArrays bigArrays, TestPhysicalOperationProviders physicalOperationProviders) { - return (physicalPlan, listener) -> executeSubPlan(bigArrays, physicalOperationProviders, physicalPlan, listener); + PlanRunner planRunner(BigArrays bigArrays, FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders) { + return (physicalPlan, listener) -> executeSubPlan(bigArrays, foldCtx, physicalOperationProviders, physicalPlan, listener); } void executeSubPlan( BigArrays bigArrays, + FoldContext foldCtx, TestPhysicalOperationProviders physicalOperationProviders, PhysicalPlan physicalPlan, ActionListener listener @@ -633,12 +637,17 @@ void executeSubPlan( // replace fragment inside the coordinator plan List drivers = new ArrayList<>(); - LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan(new OutputExec(coordinatorPlan, collectedPages::add)); + LocalExecutionPlan coordinatorNodeExecutionPlan = executionPlanner.plan( + foldCtx, + new OutputExec(coordinatorPlan, collectedPages::add) + ); drivers.addAll(coordinatorNodeExecutionPlan.createDrivers(getTestName())); if (dataNodePlan != null) { var searchStats = new DisabledSearchStats(); - var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, searchStats)); - var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(configuration, searchStats)); + var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(configuration, foldCtx, searchStats)); + var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(configuration, foldCtx, searchStats) + ); var csvDataNodePhysicalPlan = PlannerUtils.localPlan(dataNodePlan, logicalTestOptimizer, physicalTestOptimizer); exchangeSource.addRemoteSink( @@ -649,7 +658,7 @@ void executeSubPlan( throw new AssertionError("expected no failure", e); }) ); - LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(csvDataNodePhysicalPlan); + LocalExecutionPlan dataNodeExecutionPlan = executionPlanner.plan(foldCtx, csvDataNodePhysicalPlan); drivers.addAll(dataNodeExecutionPlan.createDrivers(getTestName())); Randomness.shuffle(drivers); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index dc4120f357725..2df6e30e96081 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -1077,7 +1077,7 @@ public void testImplicitLimit() { from test """); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT)); as(limit.child(), EsRelation.class); } @@ -1085,7 +1085,7 @@ public void testImplicitMaxLimitAfterLimit() { for (int i = -1; i <= 1; i++) { var plan = analyze("from test | limit " + (MAX_LIMIT + i)); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); limit = as(limit.child(), Limit.class); as(limit.child(), EsRelation.class); } @@ -1102,7 +1102,7 @@ public void testImplicitMaxLimitAfterLimitAndNonLimit() { for (int i = -1; i <= 1; i++) { var plan = analyze("from test | limit " + (MAX_LIMIT + i) + " | eval s = salary * 10 | where s > 0"); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); var filter = as(limit.child(), Filter.class); var eval = as(filter.child(), Eval.class); limit = as(eval.child(), Limit.class); @@ -1114,7 +1114,7 @@ public void testImplicitDefaultLimitAfterLimitAndBreaker() { for (var breaker : List.of("stats c = count(salary) by last_name", "sort salary")) { var plan = analyze("from test | limit 100000 | " + breaker); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(MAX_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(MAX_LIMIT)); } } @@ -1122,7 +1122,7 @@ public void testImplicitDefaultLimitAfterBreakerAndNonBreakers() { for (var breaker : List.of("stats c = count(salary) by last_name", "eval c = salary | sort c")) { var plan = analyze("from test | " + breaker + " | eval cc = c * 10 | where cc > 0"); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(DEFAULT_LIMIT)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(DEFAULT_LIMIT)); } } @@ -1428,7 +1428,7 @@ public void testEmptyEsRelationOnLimitZeroWithCount() throws IOException { var plan = analyzeWithEmptyFieldCapsResponse(query); var limit = as(plan, Limit.class); limit = as(limit.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(0)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(0)); var orderBy = as(limit.child(), OrderBy.class); var agg = as(orderBy.child(), Aggregate.class); assertEmptyEsRelation(agg.child()); @@ -1443,7 +1443,7 @@ public void testEmptyEsRelationOnConstantEvalAndKeep() throws IOException { var plan = analyzeWithEmptyFieldCapsResponse(query); var limit = as(plan, Limit.class); limit = as(limit.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(2)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(2)); var project = as(limit.child(), EsqlProject.class); var eval = as(project.child(), Eval.class); assertEmptyEsRelation(eval.child()); @@ -1460,7 +1460,7 @@ public void testEmptyEsRelationOnConstantEvalAndStats() throws IOException { var agg = as(limit.child(), Aggregate.class); var eval = as(agg.child(), Eval.class); limit = as(eval.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(10)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(10)); assertEmptyEsRelation(limit.child()); } @@ -2054,10 +2054,10 @@ public void testLookup() { } LogicalPlan plan = analyze(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000)); var lookup = as(limit.child(), Lookup.class); - assertThat(lookup.tableName().fold(), equalTo("int_number_names")); + assertThat(as(lookup.tableName(), Literal.class).value(), equalTo("int_number_names")); assertMap(lookup.matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}"))); assertThat( lookup.localRelation().output().stream().map(Object::toString).toList(), @@ -2336,7 +2336,7 @@ public void testCoalesceWithMixedNumericTypes() { projection = as(projections.get(3), ReferenceAttribute.class); assertEquals(projection.name(), "w"); assertEquals(projection.dataType(), DataType.DOUBLE); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(as(limit.limit(), Literal.class).value(), equalTo(1000)); } public void testNamedParamsForIdentifiers() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java new file mode 100644 index 0000000000000..828f9e061686b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/evaluator/mapper/EvaluatorMapperTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.evaluator.mapper; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; +import org.hamcrest.Matchers; + +public class EvaluatorMapperTests extends ESTestCase { + public void testFoldCompletesWithPlentyOfMemory() { + Add add = new Add( + Source.synthetic("shouldn't break"), + new Literal(Source.EMPTY, 1, DataType.INTEGER), + new Literal(Source.EMPTY, 3, DataType.INTEGER) + ); + assertEquals(add.fold(new FoldContext(100)), 4); + } + + public void testFoldBreaksWithLittleMemory() { + Add add = new Add( + Source.synthetic("should break"), + new Literal(Source.EMPTY, 1, DataType.INTEGER), + new Literal(Source.EMPTY, 3, DataType.INTEGER) + ); + Exception e = expectThrows(FoldContext.FoldTooMuchMemoryException.class, () -> add.fold(new FoldContext(10))); + assertThat( + e.getMessage(), + Matchers.equalTo( + "line -1:-1: Folding query used more than 10b. " + + "The expression that pushed past the limit is [should break] which needed 32b." + ) + ); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java index c086245d6fd61..87ea6315d4f3b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractAggregationTestCase.java @@ -22,6 +22,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -40,6 +41,7 @@ import java.util.stream.IntStream; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -263,12 +265,12 @@ private void evaluate(Expression evaluableExpression) { assertTrue(evaluableExpression.foldable()); if (testCase.foldingExceptionClass() != null) { - Throwable t = expectThrows(testCase.foldingExceptionClass(), evaluableExpression::fold); + Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> evaluableExpression.fold(FoldContext.small())); assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage())); return; } - Object result = evaluableExpression.fold(); + Object result = evaluableExpression.fold(FoldContext.small()); // Decode unsigned longs into BigIntegers if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) { result = NumericUtils.unsignedLongAsBigInteger((Long) result); @@ -289,7 +291,7 @@ private void resolveExpression(Expression expression, Consumer onAgg expression = resolveSurrogates(expression); // As expressions may be composed of multiple functions, we need to fold nulls bottom-up - expression = expression.transformUp(e -> new FoldNull().rule(e)); + expression = expression.transformUp(e -> new FoldNull().rule(e, unboundLogicalOptimizerContext())); assertThat(expression.dataType(), equalTo(testCase.expectedType())); Expression.TypeResolution resolution = expression.typeResolved(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 03b9dba298951..1cf087cf55ccd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; @@ -44,6 +45,7 @@ import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.evaluator.EvalMapper; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; @@ -97,6 +99,7 @@ import static java.util.Map.entry; import static org.elasticsearch.compute.data.BlockUtils.toJavaObject; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.endsWith; @@ -530,14 +533,28 @@ protected final Expression buildLiteralExpression(TestCaseSupplier.TestCase test return build(testCase.getSource(), testCase.getDataAsLiterals()); } + public static EvaluatorMapper.ToEvaluator toEvaluator() { + return new EvaluatorMapper.ToEvaluator() { + @Override + public ExpressionEvaluator.Factory apply(Expression expression) { + return evaluator(expression); + } + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); + } + }; + } + /** * Convert an {@link Expression} tree into a {@link ExpressionEvaluator.Factory} * for {@link ExpressionEvaluator}s in the same way as our planner. */ public static ExpressionEvaluator.Factory evaluator(Expression e) { - e = new FoldNull().rule(e); + e = new FoldNull().rule(e, unboundLogicalOptimizerContext()); if (e.foldable()) { - e = new Literal(e.source(), e.fold(), e.dataType()); + e = new Literal(e.source(), e.fold(FoldContext.small()), e.dataType()); } Layout.Builder builder = new Layout.Builder(); buildLayout(builder, e); @@ -545,7 +562,7 @@ public static ExpressionEvaluator.Factory evaluator(Expression e) { if (resolution.unresolved()) { throw new AssertionError("expected resolved " + resolution.message()); } - return EvalMapper.toEvaluator(e, builder.build()); + return EvalMapper.toEvaluator(FoldContext.small(), e, builder.build()); } protected final Page row(List values) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 65b9c447170f4..944515e54af75 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -20,6 +20,7 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.indices.CrankyCircuitBreakerService; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull; @@ -38,6 +39,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -132,7 +134,7 @@ public final void testEvaluate() { if (resolution.unresolved()) { throw new AssertionError("expected resolved " + resolution.message()); } - expression = new FoldNull().rule(expression); + expression = new FoldNull().rule(expression, unboundLogicalOptimizerContext()); assertThat(expression.dataType(), equalTo(testCase.expectedType())); logger.info("Result type: " + expression.dataType()); @@ -363,11 +365,11 @@ public void testFold() { return; } assertFalse("expected resolved", expression.typeResolved().unresolved()); - Expression nullOptimized = new FoldNull().rule(expression); + Expression nullOptimized = new FoldNull().rule(expression, unboundLogicalOptimizerContext()); assertThat(nullOptimized.dataType(), equalTo(testCase.expectedType())); assertTrue(nullOptimized.foldable()); if (testCase.foldingExceptionClass() == null) { - Object result = nullOptimized.fold(); + Object result = nullOptimized.fold(FoldContext.small()); // Decode unsigned longs into BigIntegers if (testCase.expectedType() == DataType.UNSIGNED_LONG && result != null) { result = NumericUtils.unsignedLongAsBigInteger((Long) result); @@ -380,7 +382,7 @@ public void testFold() { assertWarnings(testCase.getExpectedWarnings()); } } else { - Throwable t = expectThrows(testCase.foldingExceptionClass(), nullOptimized::fold); + Throwable t = expectThrows(testCase.foldingExceptionClass(), () -> nullOptimized.fold(FoldContext.small())); assertThat(t.getMessage(), equalTo(testCase.foldingExceptionMessage())); } } @@ -419,7 +421,7 @@ protected static TestCaseSupplier arithmeticExceptionOverflowCase( String typeNameOverflow = dataType.typeName().toLowerCase(Locale.ROOT) + " overflow"; return new TestCaseSupplier( "<" + typeNameOverflow + ">", - List.of(dataType), + List.of(dataType, dataType), () -> new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData(lhsSupplier.get(), dataType, "lhs"), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java index 19af9892015b2..e507640c7b23c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/CheckLicenseTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -80,7 +81,9 @@ public EsqlFunctionRegistry snapshotRegistry() { var plan = parser.createStatement(esql); plan = plan.transformDown( Limit.class, - l -> Objects.equals(l.limit().fold(), 10) ? new LicensedLimit(l.source(), l.limit(), l.child(), functionLicenseFeature) : l + l -> Objects.equals(l.limit().fold(FoldContext.small()), 10) + ? new LicensedLimit(l.source(), l.limit(), l.child(), functionLicenseFeature) + : l ); return analyzer(registry, operationMode).analyze(plan); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 72d816a65e632..9bf063518d4ba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1007,6 +1007,17 @@ public static List dateCases() { return dateCases(Long.MIN_VALUE, Long.MAX_VALUE); } + /** + * Generate cases for {@link DataType#DATETIME}. + *

+ * For multi-row parameters, see {@link MultiRowTestCaseSupplier#dateCases}. + *

+ * Helper function for if you want to specify your min and max range as dates instead of longs. + */ + public static List dateCases(Instant min, Instant max) { + return dateCases(min.toEpochMilli(), max.toEpochMilli()); + } + /** * Generate cases for {@link DataType#DATETIME}. *

@@ -1045,6 +1056,19 @@ public static List dateCases(long min, long max) { return cases; } + /** + * + * @return randomized valid date formats + */ + public static List dateFormatCases() { + return List.of( + new TypedDataSupplier("", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.KEYWORD), + new TypedDataSupplier("", () -> new BytesRef(ESTestCase.randomDateFormatterPattern()), DataType.TEXT), + new TypedDataSupplier("", () -> new BytesRef("yyyy"), DataType.KEYWORD), + new TypedDataSupplier("", () -> new BytesRef("yyyy"), DataType.TEXT) + ); + } + /** * Generate cases for {@link DataType#DATE_NANOS}. * @@ -1388,7 +1412,7 @@ public static final class TestCase { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ private final String[] expectedBuildEvaluatorWarnings; @@ -1518,7 +1542,7 @@ public String[] getExpectedWarnings() { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ public String[] getExpectedBuildEvaluatorWarnings() { return expectedBuildEvaluatorWarnings; @@ -1600,7 +1624,7 @@ public TestCase withWarning(String warning) { /** * Warnings that are added by calling {@link AbstractFunctionTestCase#evaluator} - * or {@link Expression#fold()} on the expression built by this. + * or {@link Expression#fold} on the expression built by this. */ public TestCase withBuildEvaluatorWarning(String warning) { return new TestCase( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java index de84086e3cb4e..911878a645b42 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseExtraTests.java @@ -19,9 +19,11 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.junit.After; @@ -67,7 +69,7 @@ public void testPartialFoldDropsFirstFalse() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG)))) ); } @@ -80,7 +82,7 @@ public void testPartialFoldMv() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo(new Case(Source.synthetic("case"), field("last_cond", DataType.BOOLEAN), List.of(field("last", DataType.LONG)))) ); } @@ -92,7 +94,7 @@ public void testPartialFoldNoop() { List.of(field("first", DataType.LONG), field("last", DataType.LONG)) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), sameInstance(c)); + assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c)); } public void testPartialFoldFirst() { @@ -102,7 +104,7 @@ public void testPartialFoldFirst() { List.of(field("first", DataType.LONG), field("last", DataType.LONG)) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("first", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("first", DataType.LONG))); } public void testPartialFoldFirstAfterKeepingUnknown() { @@ -118,7 +120,7 @@ public void testPartialFoldFirstAfterKeepingUnknown() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo( new Case( Source.synthetic("case"), @@ -141,7 +143,7 @@ public void testPartialFoldSecond() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG))); } public void testPartialFoldSecondAfterDroppingFalse() { @@ -156,7 +158,7 @@ public void testPartialFoldSecondAfterDroppingFalse() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("second", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("second", DataType.LONG))); } public void testPartialFoldLast() { @@ -171,7 +173,7 @@ public void testPartialFoldLast() { ) ); assertThat(c.foldable(), equalTo(false)); - assertThat(c.partiallyFold(), equalTo(field("last", DataType.LONG))); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(field("last", DataType.LONG))); } public void testPartialFoldLastAfterKeepingUnknown() { @@ -187,7 +189,7 @@ public void testPartialFoldLastAfterKeepingUnknown() { ); assertThat(c.foldable(), equalTo(false)); assertThat( - c.partiallyFold(), + c.partiallyFold(FoldContext.small()), equalTo( new Case( Source.synthetic("case"), @@ -203,7 +205,7 @@ public void testEvalCase() { DriverContext driverContext = driverContext(); Page page = new Page(driverContext.blockFactory().newConstantIntBlockWith(0, 1)); try ( - EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase::evaluator).get(driverContext); + EvalOperator.ExpressionEvaluator eval = caseExpr.toEvaluator(AbstractFunctionTestCase.toEvaluator()).get(driverContext); Block block = eval.eval(page) ) { return toJavaObject(block, 0); @@ -216,7 +218,7 @@ public void testEvalCase() { public void testFoldCase() { testCase(caseExpr -> { assertTrue(caseExpr.foldable()); - return caseExpr.fold(); + return caseExpr.fold(FoldContext.small()); }); } @@ -265,22 +267,31 @@ public void testCaseWithIncompatibleTypes() { public void testCaseIsLazy() { Case caseExpr = caseExpr(true, 1, true, 2); DriverContext driveContext = driverContext(); - EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(child -> { - Object value = child.fold(); - if (value != null && value.equals(2)) { - return dvrCtx -> new EvalOperator.ExpressionEvaluator() { - @Override - public Block eval(Page page) { - fail("Unexpected evaluation of 4th argument"); - return null; - } + EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() { + @Override + public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) { + Object value = expression.fold(FoldContext.small()); + if (value != null && value.equals(2)) { + return dvrCtx -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + fail("Unexpected evaluation of 4th argument"); + return null; + } - @Override - public void close() {} - }; + @Override + public void close() {} + }; + } + return AbstractFunctionTestCase.evaluator(expression); } - return AbstractFunctionTestCase.evaluator(child); - }).get(driveContext); + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); + } + }; + EvalOperator.ExpressionEvaluator evaluator = caseExpr.toEvaluator(toEvaluator).get(driveContext); Page page = new Page(driveContext.blockFactory().newConstantIntBlockWith(0, 1)); try (Block block = evaluator.eval(page)) { assertEquals(1, toJavaObject(block, 0)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index 05923246520fc..23a0f2307171c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; @@ -779,7 +780,7 @@ public void testFancyFolding() { return; } assertThat(e.foldable(), equalTo(true)); - Object result = e.fold(); + Object result = e.fold(FoldContext.small()); if (testCase.getExpectedBuildEvaluatorWarnings() != null) { assertWarnings(testCase.getExpectedBuildEvaluatorWarnings()); } @@ -799,18 +800,18 @@ public void testPartialFold() { } Case c = (Case) buildFieldExpression(testCase); if (extra().expectedPartialFold == null) { - assertThat(c.partiallyFold(), sameInstance(c)); + assertThat(c.partiallyFold(FoldContext.small()), sameInstance(c)); return; } if (extra().expectedPartialFold.size() == 1) { - assertThat(c.partiallyFold(), equalTo(extra().expectedPartialFold.get(0).asField())); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(extra().expectedPartialFold.get(0).asField())); return; } Case expected = build( Source.synthetic("expected"), extra().expectedPartialFold.stream().map(TestCaseSupplier.TypedData::asField).toList() ); - assertThat(c.partiallyFold(), equalTo(expected)); + assertThat(c.partiallyFold(FoldContext.small()), equalTo(expected)); } private static Function addWarnings(List warnings) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java index 9abbfbd61c1a8..8060326365d5c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatePeriodTests.java @@ -71,7 +71,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java index 6486f6efcdd3c..980eed1af9109 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToTimeDurationTests.java @@ -70,7 +70,7 @@ public static Iterable parameters() { })); } } - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java index da069e3c37cc4..b4a37b0297571 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java @@ -99,7 +99,7 @@ public static Iterable parameters() { equalTo(0) ); })); - return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } public void testDateDiffFunction() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java index be978eda06758..cd27ce511b317 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -99,7 +100,7 @@ public void testAllChronoFields() { EsqlTestUtils.TEST_CFG ); - assertThat(instance.fold(), is(date.getLong(value))); + assertThat(instance.fold(FoldContext.small()), is(date.getLong(value))); assertThat( DateExtract.process(epochMilli, new BytesRef(value.name()), EsqlTestUtils.TEST_CFG.zoneId()), is(date.getLong(value)) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java index 985f1144fbcf2..a5e6514b3e02c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatErrorTests.java @@ -28,11 +28,22 @@ protected List cases() { @Override protected Expression build(Source source, List args) { - return new DateFormat(source, args.get(0), args.get(1), EsqlTestUtils.TEST_CFG); + return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, EsqlTestUtils.TEST_CFG); } @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + // Single argument version + String source = sourceForSignature(signature); + String name = signature.get(0).typeName(); + if (signature.size() == 1) { + return equalTo("first argument of [" + source + "] must be [datetime], found value [] type [" + name + "]"); + } + // Two argument version + // Handle the weird case where we're calling the two argument version with the date first instead of the format. + if (signature.get(0).isDate()) { + return equalTo("first argument of [" + source + "] must be [string], found value [] type [" + name + "]"); + } return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "datetime"; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java index 8dfdd1ba486c7..3dd1f3e629da4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatTests.java @@ -11,18 +11,21 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase; import org.elasticsearch.xpack.esql.session.Configuration; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; +import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.matchesPattern; public class DateFormatTests extends AbstractConfigurationFunctionTestCase { public DateFormatTests(@Name("TestCase") Supplier testCaseSupplier) { @@ -31,39 +34,35 @@ public DateFormatTests(@Name("TestCase") Supplier tes @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors( - true, - List.of( - new TestCaseSupplier( - List.of(DataType.KEYWORD, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.KEYWORD, "formatter"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val") - ), - "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]", - DataType.KEYWORD, - equalTo(BytesRefs.toBytesRef("2023")) - ) + List suppliers = new ArrayList<>(); + // Formatter supplied cases + suppliers.addAll( + TestCaseSupplier.forBinaryNotCasting( + (format, value) -> new BytesRef( + DateFormatter.forPattern(((BytesRef) format).utf8ToString()).formatMillis(((Instant) value).toEpochMilli()) ), - new TestCaseSupplier( - List.of(DataType.TEXT, DataType.DATETIME), - () -> new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(new BytesRef("yyyy"), DataType.TEXT, "formatter"), - new TestCaseSupplier.TypedData(1687944333000L, DataType.DATETIME, "val") - ), - "DateFormatEvaluator[val=Attribute[channel=1], formatter=Attribute[channel=0], locale=en_US]", - DataType.KEYWORD, - equalTo(BytesRefs.toBytesRef("2023")) - ) - ) + DataType.KEYWORD, + TestCaseSupplier.dateFormatCases(), + TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), + matchesPattern("DateFormatEvaluator\\[val=Attribute\\[channel=1], formatter=Attribute\\[(channel=0|\\w+)], locale=en_US]"), + (lhs, rhs) -> List.of(), + false ) ); + // Default formatter cases + TestCaseSupplier.unary( + suppliers, + "DateFormatConstantEvaluator[val=Attribute[channel=0], formatter=format[strict_date_optional_time] locale[]]", + TestCaseSupplier.dateCases(Instant.parse("1900-01-01T00:00:00.00Z"), Instant.parse("9999-12-31T00:00:00.00Z")), + DataType.KEYWORD, + (value) -> new BytesRef(EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER.formatMillis(((Instant) value).toEpochMilli())), + List.of() + ); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override protected Expression buildWithConfiguration(Source source, List args, Configuration configuration) { - return new DateFormat(source, args.get(0), args.get(1), configuration); + return new DateFormat(source, args.get(0), args.size() == 2 ? args.get(1) : null, configuration); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java index ce6ee1702ee66..c667747a8ba75 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java @@ -32,7 +32,8 @@ public NowTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData( + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, List.of( new TestCaseSupplier( "Now Test", @@ -44,7 +45,8 @@ public static Iterable parameters() { equalTo(EsqlTestUtils.TEST_CFG.now().toInstant().toEpochMilli()) ) ) - ) + ), + (valid, position) -> "" ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java index 278c9123e30b1..6531e7bee90ab 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AcosTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java index 04fec5a20b438..410dc61ec5fa6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/AsinTests.java @@ -56,7 +56,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java index bfe35a08b8ba1..d702e28baf9d8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/CbrtTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { ); suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java index 50ed71262e5df..f3922a355180d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ETests.java @@ -29,14 +29,22 @@ public ETests(@Name("TestCase") Supplier testCaseSupp @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("E Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=2.718281828459045]", - DataType.DOUBLE, - equalTo(Math.E) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "E Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=2.718281828459045]", + DataType.DOUBLE, + equalTo(Math.E) + ) + ) + ), + (v, p) -> "" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java index d42f4ffde0609..bc5faf1b2560d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/ExpTests.java @@ -75,7 +75,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java index 44ad4547481d6..7942320656f3f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/Log10Tests.java @@ -124,7 +124,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java index 671cffe9e7f9e..0ee277dbcadb2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/LogTests.java @@ -187,11 +187,7 @@ public static Iterable parameters() { ) ); - // Add null cases before the rest of the error cases, so messages are correct. - suppliers = anyNullIsNull(true, suppliers); - - // Negative cases - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java index e93bc7b43d98e..79742952dbf59 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PiTests.java @@ -29,14 +29,22 @@ public PiTests(@Name("TestCase") Supplier testCaseSup @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Pi Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=3.141592653589793]", - DataType.DOUBLE, - equalTo(Math.PI) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "Pi Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=3.141592653589793]", + DataType.DOUBLE, + equalTo(Math.PI) + ) + ) + ), + (v, p) -> "numeric" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java index 9d8b87bab8878..2fc139a5458c3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/PowTests.java @@ -77,7 +77,7 @@ public static Iterable parameters() { ) ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java index 8c612e5e664e0..4bf1351969d79 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SignumTests.java @@ -72,7 +72,7 @@ public static Iterable parameters() { suppliers = anyNullIsNull(true, suppliers); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java index 23f2adc6c02e0..7cba5d6d57d45 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/SqrtTests.java @@ -109,7 +109,7 @@ public static Iterable parameters() { "Line -1:-1: java.lang.ArithmeticException: Square root of negative" ) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(suppliers, (v, p) -> "numeric")); + return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "numeric"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java index 1a622b25b3353..40e66333f953e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/TauTests.java @@ -29,14 +29,22 @@ public TauTests(@Name("TestCase") Supplier testCaseSu @ParametersFactory public static Iterable parameters() { - return parameterSuppliersFromTypedData(List.of(new TestCaseSupplier("Tau Test", List.of(DataType.INTEGER), () -> { - return new TestCaseSupplier.TestCase( - List.of(new TestCaseSupplier.TypedData(1, DataType.INTEGER, "foo")), - "LiteralsEvaluator[lit=6.283185307179586]", - DataType.DOUBLE, - equalTo(Tau.TAU) - ); - }))); + return parameterSuppliersFromTypedDataWithDefaultChecks( + true, + List.of( + new TestCaseSupplier( + "Tau Test", + List.of(), + () -> new TestCaseSupplier.TestCase( + List.of(), + "LiteralsEvaluator[lit=6.283185307179586]", + DataType.DOUBLE, + equalTo(Tau.TAU) + ) + ) + ), + (v, p) -> "numeric" + ); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java index 156fc4bfe7c36..0c905b28ac931 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvPSeriesWeightedSumTests.java @@ -32,22 +32,20 @@ public MvPSeriesWeightedSumTests(@Name("TestCase") Supplier parameters() { List cases = new ArrayList<>(); + doubles(cases); - cases = randomizeBytesRefsOffset(cases); - cases = anyNullIsNull( - cases, + return parameterSuppliersFromTypedDataWithDefaultChecks( (nullPosition, nullValueDataType, original) -> nullValueDataType == DataType.NULL ? DataType.NULL : original.expectedType(), (nullPosition, nullData, original) -> { if (nullData.isForceLiteral()) { return equalTo("LiteralsEvaluator[lit=null]"); } return nullData.type() == DataType.NULL ? equalTo("LiteralsEvaluator[lit=null]") : original; - } + }, + cases, + (valid, position) -> "double" ); - cases = errorsForCasesWithoutExamples(cases, (valid, position) -> "double"); - - return parameterSuppliersFromTypedData(cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java index e5f240c811bd0..5d21c135e467e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortTests.java @@ -18,9 +18,11 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matchers; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.function.Supplier; @@ -42,6 +44,12 @@ public static Iterable parameters() { doubles(suppliers); bytesRefs(suppliers); nulls(suppliers); + + suppliers = anyNullIsNull( + suppliers, + (nullPosition, nullValueDataType, original) -> nullPosition == 0 ? nullValueDataType : original.expectedType(), + (nullPosition, nullData, original) -> nullData.isForceLiteral() ? Matchers.equalTo("LiteralsEvaluator[lit=null]") : original + ); return parameterSuppliersFromTypedData(suppliers); } @@ -143,75 +151,22 @@ private static void doubles(List suppliers) { } private static void bytesRefs(List suppliers) { - suppliers.add(new TestCaseSupplier(List.of(DataType.KEYWORD, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.KEYWORD).value()); - BytesRef order = new BytesRef("DESC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.KEYWORD, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=false]", - DataType.KEYWORD, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.TEXT, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.TEXT).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.TEXT, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.TEXT, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.SEMANTIC_TEXT, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.SEMANTIC_TEXT).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.SEMANTIC_TEXT, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.SEMANTIC_TEXT, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.IP, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.IP).value()); - BytesRef order = new BytesRef("DESC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.IP, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=false]", - DataType.IP, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted(Collections.reverseOrder()).toList()) - ); - })); - - suppliers.add(new TestCaseSupplier(List.of(DataType.VERSION, DataType.KEYWORD), () -> { - List field = randomList(1, 10, () -> randomLiteral(DataType.VERSION).value()); - BytesRef order = new BytesRef("ASC"); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(field, DataType.VERSION, "field"), - new TestCaseSupplier.TypedData(order, DataType.KEYWORD, "order").forceLiteral() - ), - "MvSortBytesRef[field=Attribute[channel=0], order=true]", - DataType.VERSION, - equalTo(field.size() == 1 ? field.iterator().next() : field.stream().sorted().toList()) - ); - })); + for (DataType type : List.of(DataType.KEYWORD, DataType.TEXT, DataType.SEMANTIC_TEXT, DataType.IP, DataType.VERSION)) { + suppliers.add(new TestCaseSupplier(List.of(type, DataType.KEYWORD), () -> { + List field = randomList(1, 10, () -> (BytesRef) randomLiteral(type).value()); + boolean order = randomBoolean(); + var sortedData = field.stream().sorted(order ? Comparator.naturalOrder() : Comparator.reverseOrder()).toList(); + return new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData(field, type, "field"), + new TestCaseSupplier.TypedData(new BytesRef(order ? "ASC" : "DESC"), DataType.KEYWORD, "order").forceLiteral() + ), + "MvSortBytesRef[field=Attribute[channel=0], order=" + order + "]", + type, + equalTo(sortedData.size() == 1 ? sortedData.get(0) : sortedData) + ); + })); + } } private static void nulls(List suppliers) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java index 26d113359a6f8..89b148144fc83 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumTests.java @@ -65,7 +65,7 @@ public static Iterable parameters() { data.add(asLongUnsigned(UNSIGNED_LONG_MAX)); return data; })); - return parameterSuppliersFromTypedData(anyNullIsNull(false, cases)); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, cases, (v, p) -> "numeric"); } private static TestCaseSupplier arithmeticExceptionCase(DataType dataType, Supplier dataSupplier) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java index 797c99992815e..688341ebaa2b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -187,19 +188,27 @@ public void testCoalesceIsLazy() { Layout.Builder builder = new Layout.Builder(); buildLayout(builder, exp); Layout layout = builder.build(); - EvaluatorMapper.ToEvaluator toEvaluator = child -> { - if (child == evil) { - return dvrCtx -> new EvalOperator.ExpressionEvaluator() { - @Override - public Block eval(Page page) { - throw new AssertionError("shouldn't be called"); - } - - @Override - public void close() {} - }; + EvaluatorMapper.ToEvaluator toEvaluator = new EvaluatorMapper.ToEvaluator() { + @Override + public EvalOperator.ExpressionEvaluator.Factory apply(Expression expression) { + if (expression == evil) { + return dvrCtx -> new EvalOperator.ExpressionEvaluator() { + @Override + public Block eval(Page page) { + throw new AssertionError("shouldn't be called"); + } + + @Override + public void close() {} + }; + } + return EvalMapper.toEvaluator(FoldContext.small(), expression, layout); + } + + @Override + public FoldContext foldCtx() { + return FoldContext.small(); } - return EvalMapper.toEvaluator(child, layout); }; try ( EvalOperator.ExpressionEvaluator eval = exp.toEvaluator(toEvaluator).get(driverContext()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java index ac060b9454564..e58819beb803c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/LocateTests.java @@ -76,8 +76,6 @@ public static Iterable parameters() { } } - suppliers = anyNullIsNull(true, suppliers); - // Here follows some non-randomized examples that we want to cover on every run suppliers.add(supplier("a tiger", "a t", null, 1)); suppliers.add(supplier("a tiger", "a", null, 1)); @@ -128,7 +126,7 @@ public static Iterable parameters() { suppliers.add(supplier("🐱Meow!🐶Woof!", "Meow!🐶Woof!", 0, 2)); suppliers.add(supplier("🐱Meow!🐶Woof!", "eow!🐶Woof!", 0, 3)); - return parameterSuppliersFromTypedData(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java index 4f8adf3abaae6..6c41552a9fc52 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RLikeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -159,8 +160,8 @@ protected Expression build(Source source, List args) { Expression expression = args.get(0); Literal pattern = (Literal) args.get(1); Literal caseInsensitive = args.size() > 2 ? (Literal) args.get(2) : null; - String patternString = ((BytesRef) pattern.fold()).utf8ToString(); - boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold() : false; + String patternString = ((BytesRef) pattern.fold(FoldContext.small())).utf8ToString(); + boolean caseInsensitiveBool = caseInsensitive != null ? (boolean) caseInsensitive.fold(FoldContext.small()) : false; logger.info("pattern={} caseInsensitive={}", patternString, caseInsensitiveBool); return caseInsensitiveBool diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java index 8b4ea066fdccb..5eb654b0d8235 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/RepeatTests.java @@ -122,13 +122,11 @@ public static Iterable parameters() { .withFoldingException(IllegalArgumentException.class, "Number parameter cannot be negative, found [" + number + "]"); })); - cases = anyNullIsNull(true, cases); - cases = errorsForCasesWithoutExamples(cases, (v, p) -> switch (p) { + return parameterSuppliersFromTypedDataWithDefaultChecks(true, cases, (v, p) -> switch (p) { case 0 -> "string"; case 1 -> "integer"; default -> ""; }); - return parameterSuppliersFromTypedData(cases); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java index 58d52cc02b548..397fb8064626c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ReverseTests.java @@ -39,7 +39,7 @@ public static Iterable parameters() { } } - return parameterSuppliersFromTypedData(suppliers); + return parameterSuppliersFromTypedDataWithDefaultChecks(false, suppliers, (v, p) -> "string"); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java index b355feb6130a3..f779dd038454d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,7 @@ public void testRandomLocale() { String testString = randomAlphaOfLength(10); Configuration cfg = randomLocaleConfig(); ToLower func = new ToLower(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg); - assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold())); + assertThat(BytesRefs.toBytesRef(testString.toLowerCase(cfg.locale())), equalTo(func.fold(FoldContext.small()))); } private Configuration randomLocaleConfig() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java index fdae4f953a0fa..3957c2e1fb2c0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -54,7 +55,7 @@ public void testRandomLocale() { String testString = randomAlphaOfLength(10); Configuration cfg = randomLocaleConfig(); ToUpper func = new ToUpper(Source.EMPTY, new Literal(Source.EMPTY, testString, DataType.KEYWORD), cfg); - assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold())); + assertThat(BytesRefs.toBytesRef(testString.toUpperCase(cfg.locale())), equalTo(func.fold(FoldContext.small()))); } private Configuration randomLocaleConfig() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java index eed2c7379e9e1..6626ac50d60b5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/WildcardLikeTests.java @@ -12,6 +12,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -78,8 +79,8 @@ protected Expression build(Source source, List args) { Literal pattern = (Literal) args.get(1); if (args.size() > 2) { Literal caseInsensitive = (Literal) args.get(2); - assertThat(caseInsensitive.fold(), equalTo(false)); + assertThat(caseInsensitive.fold(FoldContext.small()), equalTo(false)); } - return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold()).utf8ToString())); + return new WildcardLike(source, expression, new WildcardPattern(((BytesRef) pattern.fold(FoldContext.small())).utf8ToString())); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java index aa4c037e5e961..62cb0bd868ee5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddTests.java @@ -35,7 +35,6 @@ import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; @@ -247,33 +246,7 @@ public static Iterable parameters() { ); }).toList()); - // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for - // Datetime + Period/Duration, since the expression will take the non-null arg's type. - suppliers = anyNullIsNull( - suppliers, - (nullPosition, nullType, original) -> original.expectedType(), - (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original - ); - suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString); - - // Cases that should generate warnings - suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> { - // Ensure we don't have an overflow - int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs") - ), - "AddIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - DataType.INTEGER, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value"); - })); - // exact math arithmetic exceptions + // Exact math arithmetic exceptions suppliers.add( arithmeticExceptionOverflowCase( DataType.INTEGER, @@ -315,6 +288,16 @@ public static Iterable parameters() { ) ); + // Datetime tests are split in two, depending on their permissiveness of null-injection, which cannot happen "automatically" for + // Datetime + Period/Duration, since the expression will take the non-null arg's type. + suppliers = anyNullIsNull( + suppliers, + (nullPosition, nullType, original) -> original.expectedType(), + (nullPosition, nullData, original) -> nullData.isForceLiteral() ? equalTo("LiteralsEvaluator[lit=null]") : original + ); + suppliers = errorsForCasesWithoutExamples(suppliers, AddTests::addErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java index c6b607ded7999..70536cd0036e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivTests.java @@ -89,8 +89,6 @@ public static Iterable parameters() { ) ); - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString); - // Divide by zero cases - all of these should warn and return null TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( new TestCaseSupplier.NumericTypeTestConfig<>( @@ -163,6 +161,9 @@ public static Iterable parameters() { ) ); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), DivTests::divErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java index 65335cdeeb4e1..2ef1cbe89e6a2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModTests.java @@ -76,8 +76,6 @@ public static Iterable parameters() { ) ); - suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString); - // Divide by zero cases - all of these should warn and return null TestCaseSupplier.NumericTypeTestConfigs typeStuff = new TestCaseSupplier.NumericTypeTestConfigs<>( new TestCaseSupplier.NumericTypeTestConfig<>( @@ -150,6 +148,9 @@ public static Iterable parameters() { ) ); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), ModTests::modErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java index a2ca331cb261e..12b685eaeeb8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulTests.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.function.Supplier; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; @@ -124,7 +125,20 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(false, suppliers)); + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers), MulTests::mulErrorMessageString); + + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial + return parameterSuppliersFromTypedData(suppliers); + } + + private static String mulErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "numeric"); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + return "[*] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; + + } } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java index a8c7b5b5a83fd..15860d35539e0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/NegTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -154,7 +155,7 @@ public void testEdgeCases() { private Object foldTemporalAmount(Object val) { Neg neg = new Neg(Source.EMPTY, new Literal(Source.EMPTY, val, typeOf(val))); - return neg.fold(); + return neg.fold(FoldContext.small()); } private static DataType typeOf(Object val) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java index 3a7d4e6482d72..600db229e90a3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubTests.java @@ -24,6 +24,7 @@ import java.time.ZonedDateTime; import java.time.temporal.TemporalAmount; import java.util.List; +import java.util.Set; import java.util.function.BinaryOperator; import java.util.function.Supplier; import java.util.function.ToLongBiFunction; @@ -33,8 +34,6 @@ import static org.elasticsearch.xpack.esql.core.util.DateUtils.asMillis; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.ZERO_AS_UNSIGNED_LONG; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; public class SubTests extends AbstractScalarFunctionTestCase { @@ -245,7 +244,8 @@ public static Iterable parameters() { "SubUnsignedLongsEvaluator" ) ); - suppliers = anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> { + + suppliers = errorsForCasesWithoutExamples(anyNullIsNull(suppliers, (nullPosition, nullValueDataType, original) -> { if (nullValueDataType == DataType.NULL) { return original.getData().get(nullPosition == 0 ? 1 : 0).type(); } @@ -255,28 +255,30 @@ public static Iterable parameters() { return equalTo("LiteralsEvaluator[lit=null]"); } return original; - }); - - suppliers.add(new TestCaseSupplier("MV", List.of(DataType.INTEGER, DataType.INTEGER), () -> { - // Ensure we don't have an overflow - int rhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - int lhs2 = randomIntBetween((Integer.MIN_VALUE >> 1) - 1, (Integer.MAX_VALUE >> 1) - 1); - return new TestCaseSupplier.TestCase( - List.of( - new TestCaseSupplier.TypedData(List.of(lhs, lhs2), DataType.INTEGER, "lhs"), - new TestCaseSupplier.TypedData(rhs, DataType.INTEGER, "rhs") - ), - "SubIntsEvaluator[lhs=Attribute[channel=0], rhs=Attribute[channel=1]]", - DataType.INTEGER, - is(nullValue()) - ).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.") - .withWarning("Line -1:-1: java.lang.IllegalArgumentException: single-value function encountered multi-value"); - })); + }), SubTests::subErrorMessageString); + // Cannot use parameterSuppliersFromTypedDataWithDefaultChecks as error messages are non-trivial return parameterSuppliersFromTypedData(suppliers); } + private static String subErrorMessageString(boolean includeOrdinal, List> validPerPosition, List types) { + if (types.get(1) == DataType.DATETIME) { + if (types.get(0).isNumeric() || DataType.isMillisOrNanos(types.get(0))) { + return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [datetime]"; + } + if (DataType.isNull(types.get(0))) { + return "[-] arguments are in unsupported order: cannot subtract a [DATETIME] value [datetime] from a [NULL] amount [null]"; + } + } + + try { + return typeErrorMessage(includeOrdinal, validPerPosition, types, (a, b) -> "date_nanos, datetime or numeric"); + } catch (IllegalStateException e) { + // This means all the positional args were okay, so the expected error is from the combination + return "[-] has arguments with incompatible types [" + types.get(0).typeName() + "] and [" + types.get(1).typeName() + "]"; + } + } + @Override protected Expression build(Source source, List args) { return new Sub(source, args.get(0), args.get(1)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java index 8e46f7d28540d..4b9f21187f425 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsTests.java @@ -236,7 +236,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java index f00e8a19f9dac..45cfef518d39a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualTests.java @@ -158,7 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java index 169f1886af47f..6ab55ddbcf403 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanTests.java @@ -172,7 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java index b004adca351ab..80f67ec8e5e3a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -48,27 +49,27 @@ public InTests(@Name("TestCase") Supplier testCaseSup public void testInWithContainedValue() { In in = new In(EMPTY, TWO, Arrays.asList(ONE, TWO, THREE)); - assertTrue((Boolean) in.fold()); + assertTrue((Boolean) in.fold(FoldContext.small())); } public void testInWithNotContainedValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, TWO)); - assertFalse((Boolean) in.fold()); + assertFalse((Boolean) in.fold(FoldContext.small())); } public void testHandleNullOnLeftValue() { In in = new In(EMPTY, NULL, Arrays.asList(ONE, TWO, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); in = new In(EMPTY, NULL, Arrays.asList(ONE, NULL, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); } public void testHandleNullsOnRightValue() { In in = new In(EMPTY, THREE, Arrays.asList(ONE, NULL, THREE)); - assertTrue((Boolean) in.fold()); + assertTrue((Boolean) in.fold(FoldContext.small())); in = new In(EMPTY, ONE, Arrays.asList(TWO, NULL, THREE)); - assertNull(in.fold()); + assertNull(in.fold(FoldContext.small())); } private static Literal L(Object value) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java index faf0a0d8f418c..6fa1112f23f45 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; @@ -18,37 +19,37 @@ public class InsensitiveEqualsTests extends ESTestCase { public void testFold() { - assertTrue(insensitiveEquals(l("foo"), l("foo")).fold()); - assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold()); - assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold()); - assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold()); - assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold()); - assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold()); - assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold()); - assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold()); - assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold()); - assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold()); - assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold()); - assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("*")).fold()); + assertTrue(insensitiveEquals(l("foo"), l("foo")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("Foo"), l("foo")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("Foo"), l("fOO")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo*"), l("foo*")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo*"), l("FOO*")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo?bar"), l("foo?bar")).fold(FoldContext.small())); + assertTrue(insensitiveEquals(l("foo?bar"), l("FOO?BAR")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Foo"), l("fo*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Fox"), l("fo?")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("Foo"), l("*OO")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("BarFooBaz"), l("*O*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("BarFooBaz"), l("bar*baz")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("*")).fold(FoldContext.small())); - assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold()); - assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold()); - assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold()); + assertFalse(insensitiveEquals(l("foo*bar"), l("foo\\*bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo?"), l("foo\\?")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo?bar"), l("foo\\?bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(10)), l("*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("???")).fold(FoldContext.small())); - assertFalse(insensitiveEquals(l("foo"), l("bar")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold()); - assertFalse(insensitiveEquals(l(""), l("bar")).fold()); - assertFalse(insensitiveEquals(l("foo"), l("")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold()); - assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold()); + assertFalse(insensitiveEquals(l("foo"), l("bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("ba*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("*a*")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(""), l("bar")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l("foo"), l("")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("??")).fold(FoldContext.small())); + assertFalse(insensitiveEquals(l(randomAlphaOfLength(3)), l("????")).fold(FoldContext.small())); - assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold()); - assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold()); - assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold()); + assertNull(insensitiveEquals(l("foo"), Literal.NULL).fold(FoldContext.small())); + assertNull(insensitiveEquals(Literal.NULL, l("foo")).fold(FoldContext.small())); + assertNull(insensitiveEquals(Literal.NULL, Literal.NULL).fold(FoldContext.small())); } public void testProcess() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java index 7eccc2c8cc86b..8d48d5812d9da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualTests.java @@ -158,7 +158,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java index a17be8aa97978..f5f4bae5b2afe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java @@ -172,7 +172,7 @@ public static Iterable parameters() { ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java index 296b4b37cfa73..95001366f1cad 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsTests.java @@ -232,7 +232,8 @@ public static Iterable parameters() { false ) ); - return parameterSuppliersFromTypedData(anyNullIsNull(true, suppliers)); + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 0c03556241d28..11cd123c731e8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; @@ -70,6 +71,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForExistingField; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; @@ -93,7 +95,7 @@ public static void init() { mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()), @@ -161,7 +163,7 @@ public void testMissingFieldInProject() { assertThat(Expressions.names(eval.fields()), contains("last_name")); var alias = as(eval.fields().get(0), Alias.class); var literal = as(alias.child(), Literal.class); - assertThat(literal.fold(), is(nullValue())); + assertThat(literal.value(), is(nullValue())); assertThat(literal.dataType(), is(DataType.KEYWORD)); var limit = as(eval.child(), Limit.class); @@ -304,7 +306,7 @@ public void testMissingFieldInEval() { var alias = as(eval.fields().get(0), Alias.class); var literal = as(alias.child(), Literal.class); - assertThat(literal.fold(), is(nullValue())); + assertThat(literal.value(), is(nullValue())); assertThat(literal.dataType(), is(DataType.INTEGER)); var limit = as(eval.child(), Limit.class); @@ -402,7 +404,7 @@ public void testSparseDocument() throws Exception { EsIndex index = new EsIndex("large", large, Map.of("large", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(index); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, EsqlTestUtils.emptyPolicyResolution()), @@ -411,7 +413,7 @@ public void testSparseDocument() throws Exception { var analyzed = analyzer.analyze(parser.createStatement(query)); var optimized = logicalOptimizer.optimize(analyzed); - var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats); + var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); var plan = new LocalLogicalPlanOptimizer(localContext).localOptimize(optimized); var project = as(plan, Project.class); @@ -423,7 +425,7 @@ public void testSparseDocument() throws Exception { var eval = as(project.child(), Eval.class); var field = eval.fields().get(0); assertThat(Expressions.name(field), is("field005")); - assertThat(Alias.unwrap(field).fold(), Matchers.nullValue()); + assertThat(Alias.unwrap(field).fold(FoldContext.small()), Matchers.nullValue()); } // InferIsNotNull @@ -561,7 +563,7 @@ private LogicalPlan plan(String query) { } private LogicalPlan localPlan(LogicalPlan plan, SearchStats searchStats) { - var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, searchStats); + var localContext = new LocalLogicalOptimizerContext(EsqlTestUtils.TEST_CFG, FoldContext.small(), searchStats); // System.out.println(plan); var localPlan = new LocalLogicalPlanOptimizer(localContext).localOptimize(plan); // System.out.println(localPlan); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 6dee34323443d..1536ed7f99fec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -90,6 +91,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; @@ -407,7 +409,7 @@ public void testMultiCountAllWithFilter() { @SuppressWarnings("unchecked") public void testSingleCountWithStatsFilter() { // an optimizer that filters out the ExtractAggregateCommonFilter rule - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config)) { + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()) { @Override protected List> batches() { var oldBatches = super.batches(); @@ -486,7 +488,7 @@ public void testQueryStringFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.queryStringQuery("last_name: Smith"); assertThat(query.query().toString(), is(expected.toString())); } @@ -515,7 +517,7 @@ public void testQueryStringFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -550,7 +552,7 @@ public void testQueryStringFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 37, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -585,7 +587,7 @@ public void testQueryStringFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -618,7 +620,7 @@ public void testQueryStringFunctionMultipleQstrClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var queryStringLeft = QueryBuilders.queryStringQuery("last_name: Smith"); var queryStringRight = QueryBuilders.queryStringQuery("emp_no: [10010 TO *]"); @@ -647,7 +649,7 @@ public void testMatchFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); assertThat(query.query().toString(), is(expected.toString())); } @@ -676,7 +678,7 @@ public void testMatchFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 38, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -711,7 +713,7 @@ public void testMatchFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 32, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -745,7 +747,7 @@ public void testMatchFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -777,7 +779,7 @@ public void testMatchFunctionMultipleMatchClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var queryStringLeft = QueryBuilders.matchQuery("last_name", "Smith").lenient(true); var queryStringRight = QueryBuilders.matchQuery("first_name", "John").lenient(true); @@ -806,7 +808,7 @@ public void testKqlFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = kqlQueryBuilder("last_name: Smith"); assertThat(query.query().toString(), is(expected.toString())); } @@ -835,7 +837,7 @@ public void testKqlFunctionConjunctionWhereOperands() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -870,7 +872,7 @@ public void testKqlFunctionWithFunctionsPushedToLucene() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(2, 36, "cidr_match(ip, \"127.0.0.1/32\")"); var terms = wrapWithSingleQuery(queryText, QueryBuilders.termsQuery("ip", "127.0.0.1/32"), "ip", filterSource); @@ -905,7 +907,7 @@ public void testKqlFunctionMultipleWhereClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); Source filterSource = new Source(3, 8, "emp_no > 10000"); var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource); @@ -938,7 +940,7 @@ public void testKqlFunctionMultipleKqlClauses() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var kqlQueryLeft = kqlQueryBuilder("last_name: Smith"); var kqlQueryRight = kqlQueryBuilder("emp_no > 10010"); @@ -1004,7 +1006,7 @@ public void testIsNotNullPushdownFilter() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.existsQuery("emp_no"); assertThat(query.query().toString(), is(expected.toString())); } @@ -1028,7 +1030,7 @@ public void testIsNullPushdownFilter() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("emp_no")); assertThat(query.query().toString(), is(expected.toString())); } @@ -1599,7 +1601,7 @@ public void testTermFunction() { var project = as(exchange.child(), ProjectExec.class); var field = as(project.child(), FieldExtractExec.class); var query = as(field.child(), EsQueryExec.class); - assertThat(query.limit().fold(), is(1000)); + assertThat(as(query.limit(), Literal.class).value(), is(1000)); var expected = QueryBuilders.termQuery("last_name", "Smith"); assertThat(query.query().toString(), is(expected.toString())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 4d175dea05071..2aed259e7ad0b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -151,6 +152,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.localSource; import static org.elasticsearch.xpack.esql.EsqlTestUtils.referenceAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; @@ -188,6 +190,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase { private static EsqlParser parser; private static Analyzer analyzer; + private static LogicalOptimizerContext logicalOptimizerCtx; private static LogicalPlanOptimizer logicalOptimizer; private static Map mapping; private static Map mappingAirports; @@ -203,7 +206,7 @@ public class LogicalPlanOptimizerTests extends ESTestCase { private static Analyzer metricsAnalyzer; private static class SubstitutionOnlyOptimizer extends LogicalPlanOptimizer { - static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + static SubstitutionOnlyOptimizer INSTANCE = new SubstitutionOnlyOptimizer(unboundLogicalOptimizerContext()); SubstitutionOnlyOptimizer(LogicalOptimizerContext optimizerContext) { super(optimizerContext); @@ -218,7 +221,8 @@ protected List> batches() { @BeforeClass public static void init() { parser = new EsqlParser(); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizerCtx = unboundLogicalOptimizerContext(); + logicalOptimizer = new LogicalPlanOptimizer(logicalOptimizerCtx); enrichResolution = new EnrichResolution(); AnalyzerTestUtils.loadEnrichPolicyResolution(enrichResolution, "languages_idx", "id", "languages_idx", "mapping-languages.json"); @@ -325,7 +329,7 @@ public void testEmptyProjectInStatWithEval() { assertThat(exprs.size(), equalTo(1)); var alias = as(exprs.get(0), Alias.class); assertThat(alias.name(), equalTo("x")); - assertThat(alias.child().fold(), equalTo(1)); + assertThat(alias.child().fold(FoldContext.small()), equalTo(1)); } /** @@ -361,11 +365,11 @@ public void testEmptyProjectInStatWithGroupAndEval() { assertThat(exprs.size(), equalTo(1)); var alias = as(exprs.get(0), Alias.class); assertThat(alias.name(), equalTo("x")); - assertThat(alias.child().fold(), equalTo(1)); + assertThat(alias.child().fold(FoldContext.small()), equalTo(1)); var filterCondition = as(filter.condition(), GreaterThan.class); assertThat(Expressions.name(filterCondition.left()), equalTo("languages")); - assertThat(filterCondition.right().fold(), equalTo(1)); + assertThat(filterCondition.right().fold(FoldContext.small()), equalTo(1)); } public void testCombineProjections() { @@ -625,7 +629,7 @@ public void testReplaceStatsFilteredAggWithEvalSingleAggWithExpression() { assertThat(alias.name(), is("sum(salary) + 1 where false")); var add = as(alias.child(), Add.class); var literal = as(add.right(), Literal.class); - assertThat(literal.fold(), is(1)); + assertThat(literal.value(), is(1)); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), LocalRelation.class); @@ -658,7 +662,7 @@ public void testReplaceStatsFilteredAggWithEvalMixedFilterAndNoFilter() { var alias = as(eval.fields().getFirst(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); alias = as(eval.fields().getLast(), Alias.class); @@ -695,7 +699,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() { var alias = as(eval.fields().getFirst(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); alias = as(eval.fields().get(1), Alias.class); @@ -703,7 +707,7 @@ public void testReplaceStatsFilteredAggWithEvalFilterFalseAndNull() { alias = as(eval.fields().getLast(), Alias.class); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(LONG)); var limit = as(eval.child(), Limit.class); @@ -752,7 +756,7 @@ public void testReplaceStatsFilteredAggWithEvalCountDistinctInExpression() { assertThat(alias.name(), is("count_distinct(salary + 2) + 3 where false")); var add = as(alias.child(), Add.class); var literal = as(add.right(), Literal.class); - assertThat(literal.fold(), is(3)); + assertThat(literal.value(), is(3)); var limit = as(eval.child(), Limit.class); var source = as(limit.child(), LocalRelation.class); @@ -788,13 +792,13 @@ public void testReplaceStatsFilteredAggWithEvalSameAggWithAndWithoutFilter() { var alias = as(eval.fields().getFirst(), Alias.class); assertThat(Expressions.name(alias), containsString("max_a")); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(INTEGER)); alias = as(eval.fields().getLast(), Alias.class); assertThat(Expressions.name(alias), containsString("min_a")); assertTrue(alias.child().foldable()); - assertThat(alias.child().fold(), nullValue()); + assertThat(alias.child().fold(FoldContext.small()), nullValue()); assertThat(alias.child().dataType(), is(INTEGER)); var limit = as(eval.child(), Limit.class); @@ -933,7 +937,7 @@ public void testExtractStatsCommonFilterUsingJustOneAlias() { var gt = as(filter.condition(), GreaterThan.class); assertThat(Expressions.name(gt.left()), is("emp_no")); assertTrue(gt.right().foldable()); - assertThat(gt.right().fold(), is(1)); + assertThat(gt.right().fold(FoldContext.small()), is(1)); var source = as(filter.child(), EsRelation.class); } @@ -1053,7 +1057,7 @@ public void testExtractStatsCommonFilterInConjunction() { var gt = as(filter.condition(), GreaterThan.class); // name is "emp_no > 1 + 1" assertThat(Expressions.name(gt.left()), is("emp_no")); assertTrue(gt.right().foldable()); - assertThat(gt.right().fold(), is(2)); + assertThat(gt.right().fold(FoldContext.small()), is(2)); var source = as(filter.child(), EsRelation.class); } @@ -1083,12 +1087,12 @@ public void testExtractStatsCommonFilterInConjunctionWithMultipleCommonConjuncti var lt = as(and.left(), LessThan.class); assertThat(Expressions.name(lt.left()), is("emp_no")); assertTrue(lt.right().foldable()); - assertThat(lt.right().fold(), is(10)); + assertThat(lt.right().fold(FoldContext.small()), is(10)); var equals = as(and.right(), Equals.class); assertThat(Expressions.name(equals.left()), is("last_name")); assertTrue(equals.right().foldable()); - assertThat(equals.right().fold(), is(BytesRefs.toBytesRef("Doe"))); + assertThat(equals.right().fold(FoldContext.small()), is(BytesRefs.toBytesRef("Doe"))); var source = as(filter.child(), EsRelation.class); } @@ -1303,7 +1307,7 @@ public void testCombineLimits() { var anotherLimit = new Limit(EMPTY, L(limitValues[secondLimit]), oneLimit); assertEquals( new Limit(EMPTY, L(Math.min(limitValues[0], limitValues[1])), emptySource()), - new PushDownAndCombineLimits().rule(anotherLimit) + new PushDownAndCombineLimits().rule(anotherLimit, logicalOptimizerCtx) ); } @@ -1322,7 +1326,7 @@ public void testPushdownLimitsPastLeftJoin() { var limit = new Limit(EMPTY, L(10), join); - var optimizedPlan = new PushDownAndCombineLimits().rule(limit); + var optimizedPlan = new PushDownAndCombineLimits().rule(limit, logicalOptimizerCtx); assertEquals(join.replaceChildren(limit.replaceChild(join.left()), join.right()), optimizedPlan); } @@ -1340,10 +1344,7 @@ public void testMultipleCombineLimits() { var value = i == limitWithMinimum ? minimum : randomIntBetween(100, 1000); plan = new Limit(EMPTY, L(value), plan); } - assertEquals( - new Limit(EMPTY, L(minimum), relation), - new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)).optimize(plan) - ); + assertEquals(new Limit(EMPTY, L(minimum), relation), logicalOptimizer.optimize(plan)); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/115311") @@ -1864,7 +1865,7 @@ public void testCopyDefaultLimitPastMvExpand() { assertThat(mvExpand.limit(), equalTo(1000)); var keep = as(mvExpand.child(), EsqlProject.class); var limitPastMvExpand = as(keep.child(), Limit.class); - assertThat(limitPastMvExpand.limit().fold(), equalTo(1000)); + assertThat(limitPastMvExpand.limit().fold(FoldContext.small()), equalTo(1000)); as(limitPastMvExpand.child(), EsRelation.class); } @@ -1887,7 +1888,7 @@ public void testDontPushDownLimitPastMvExpand() { assertThat(mvExpand.limit(), equalTo(10)); var project = as(mvExpand.child(), EsqlProject.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1)); as(limit.child(), EsRelation.class); } @@ -1921,7 +1922,7 @@ public void testMultipleMvExpandWithSortAndLimit() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var mvExp = as(topN.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(5)); @@ -1931,7 +1932,7 @@ public void testMultipleMvExpandWithSortAndLimit() { mvExp = as(filter.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(10)); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10)); filter = as(topN.child(), Filter.class); as(filter.child(), EsRelation.class); } @@ -1955,11 +1956,11 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary", "first_name")); var mvExp = as(topN.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -1985,14 +1986,14 @@ public void testPushDownLimitThroughMultipleSort_AfterMvExpand2() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("first_name")); topN = as(topN.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var mvExp = as(topN.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2021,11 +2022,11 @@ public void testDontPushDownLimitPastAggregate_AndMvExpand() { var limit = as(plan, Limit.class); var filter = as(limit.child(), Filter.class); - assertThat(limit.limit().fold(), equalTo(5)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(5)); var agg = as(filter.child(), Aggregate.class); var mvExp = as(agg.child(), MvExpand.class); var topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(50)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(50)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2052,13 +2053,13 @@ public void testPushDown_TheRightLimit_PastMvExpand() { | limit 5"""); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(5)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(5)); var filter = as(limit.child(), Filter.class); var agg = as(filter.child(), Aggregate.class); var mvExp = as(agg.child(), MvExpand.class); assertThat(mvExp.limit(), equalTo(50)); limit = as(mvExp.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(50)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(50)); as(limit.child(), EsRelation.class); } @@ -2083,12 +2084,12 @@ public void testPushDownLimit_PastEvalAndMvExpand() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(5)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(5)); assertThat(orderNames(topN), contains("salary")); var eval = as(topN.child(), Eval.class); var mvExp = as(eval.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("first_name")); as(topN.child(), EsRelation.class); } @@ -2114,7 +2115,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField_ResultT var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(1000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2143,7 +2144,7 @@ public void testFilterWithSortBeforeMvExpand() { var mvExp = as(plan, MvExpand.class); assertThat(mvExp.limit(), equalTo(10)); var topN = as(mvExp.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(10)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10)); assertThat(orderNames(topN), contains("emp_no")); var filter = as(topN.child(), Filter.class); as(filter.child(), EsRelation.class); @@ -2168,7 +2169,7 @@ public void testMultiMvExpand_SortDownBelow() { | sort first_name"""); var topN = as(plan, TopN.class); - assertThat(topN.limit().fold(), equalTo(1000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(orderNames(topN), contains("first_name")); var mvExpand = as(topN.child(), MvExpand.class); var filter = as(mvExpand.child(), Filter.class); @@ -2200,11 +2201,11 @@ public void testLimitThenSortBeforeMvExpand() { assertThat(mvExpand.limit(), equalTo(10000)); var project = as(mvExpand.child(), EsqlProject.class); var topN = as(project.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(7300)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(7300)); assertThat(orderNames(topN), contains("a")); mvExpand = as(topN.child(), MvExpand.class); var limit = as(mvExpand.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(7300)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(7300)); as(limit.child(), LocalRelation.class); } @@ -2224,7 +2225,7 @@ public void testRemoveUnusedSortBeforeMvExpand_DefaultLimit10000() { var topN = as(plan, TopN.class); assertThat(orderNames(topN), contains("first_name")); - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); var mvExpand = as(topN.child(), MvExpand.class); var topN2 = as(mvExpand.child(), TopN.class); // TODO is it correct? Double-check AddDefaultTopN rule as(topN2.child(), EsRelation.class); @@ -2252,7 +2253,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2260,7 +2261,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedField() { topN = as(mvExp.child(), TopN.class); // the filter acts on first_name (the one used in mv_expand), so the limit 15 is not pushed down past mv_expand // instead the default limit is added - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2287,7 +2288,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); @@ -2295,7 +2296,7 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilter_NOT_OnExpandedField() topN = as(mvExp.child(), TopN.class); // the filters after mv_expand do not act on the expanded field values, as such the limit 15 is the one being pushed down // otherwise that limit wouldn't have pushed down and the default limit was instead being added by default before mv_expanded - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("emp_no")); as(topN.child(), EsRelation.class); } @@ -2323,14 +2324,14 @@ public void testAddDefaultLimit_BeforeMvExpand_WithFilterOnExpandedFieldAlias() var keep = as(plan, EsqlProject.class); var topN = as(keep.child(), TopN.class); - assertThat(topN.limit().fold(), equalTo(15)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(15)); assertThat(orderNames(topN), contains("salary", "first_name")); var filter = as(topN.child(), Filter.class); assertThat(filter.condition(), instanceOf(And.class)); var mvExp = as(filter.child(), MvExpand.class); topN = as(mvExp.child(), TopN.class); // the filter uses an alias ("x") to the expanded field ("first_name"), so the default limit is used and not the one provided - assertThat(topN.limit().fold(), equalTo(10000)); + assertThat(topN.limit().fold(FoldContext.small()), equalTo(10000)); assertThat(orderNames(topN), contains("gender")); as(topN.child(), EsRelation.class); } @@ -2369,7 +2370,7 @@ public void testSortMvExpandLimit() { var expand = as(plan, MvExpand.class); assertThat(expand.limit(), equalTo(20)); var topN = as(expand.child(), TopN.class); - assertThat(topN.limit().fold(), is(20)); + assertThat(topN.limit().fold(FoldContext.small()), is(20)); var row = as(topN.child(), EsRelation.class); } @@ -2390,7 +2391,7 @@ public void testWhereMvExpand() { var expand = as(plan, MvExpand.class); assertThat(expand.limit(), equalTo(1000)); var limit2 = as(expand.child(), Limit.class); - assertThat(limit2.limit().fold(), is(1000)); + assertThat(limit2.limit().fold(FoldContext.small()), is(1000)); var row = as(limit2.child(), LocalRelation.class); } @@ -2583,7 +2584,7 @@ public void testSimplifyLikeNoWildcard() { assertTrue(filter.condition() instanceof Equals); Equals equals = as(filter.condition(), Equals.class); - assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold()); + assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small())); assertTrue(filter.child() instanceof EsRelation); } @@ -2609,7 +2610,7 @@ public void testSimplifyRLikeNoWildcard() { assertTrue(filter.condition() instanceof Equals); Equals equals = as(filter.condition(), Equals.class); - assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold()); + assertEquals(BytesRefs.toBytesRef("foo"), equals.right().fold(FoldContext.small())); assertTrue(filter.child() instanceof EsRelation); } @@ -2773,7 +2774,7 @@ public void testEnrich() { """); var enrich = as(plan, Enrich.class); assertTrue(enrich.policyName().resolved()); - assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx"))); + assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx"))); var eval = as(enrich.child(), Eval.class); var limit = as(eval.child(), Limit.class); as(limit.child(), EsRelation.class); @@ -2819,7 +2820,7 @@ public void testEnrichNotNullFilter() { var filter = as(limit.child(), Filter.class); var enrich = as(filter.child(), Enrich.class); assertTrue(enrich.policyName().resolved()); - assertThat(enrich.policyName().fold(), is(BytesRefs.toBytesRef("languages_idx"))); + assertThat(enrich.policyName().fold(FoldContext.small()), is(BytesRefs.toBytesRef("languages_idx"))); var eval = as(enrich.child(), Eval.class); as(eval.child(), EsRelation.class); } @@ -2940,7 +2941,7 @@ public void testMedianReplacement() { var a = as(aggs.get(0), Alias.class); var per = as(a.child(), Percentile.class); var literal = as(per.percentile(), Literal.class); - assertThat((int) QuantileStates.MEDIAN, is(literal.fold())); + assertThat((int) QuantileStates.MEDIAN, is(literal.value())); assertThat(Expressions.names(agg.groupings()), contains("last_name")); } @@ -2949,7 +2950,7 @@ public void testSplittingInWithFoldableValue() { FieldAttribute fa = getFieldAttribute("foo"); In in = new In(EMPTY, ONE, List.of(TWO, THREE, fa, L(null))); Or expected = new Or(EMPTY, new In(EMPTY, ONE, List.of(TWO, THREE)), new In(EMPTY, ONE, List.of(fa, L(null)))); - assertThat(new SplitInWithFoldableValue().rule(in), equalTo(expected)); + assertThat(new SplitInWithFoldableValue().rule(in, logicalOptimizerCtx), equalTo(expected)); } public void testReplaceFilterWithExact() { @@ -3706,7 +3707,7 @@ private void aggFieldName(Expression exp, Class var alias = as(exp, Alias.class); var af = as(alias.child(), aggType); var field = af.field(); - var name = field.foldable() ? BytesRefs.toString(field.fold()) : Expressions.name(field); + var name = field.foldable() ? BytesRefs.toString(field.fold(FoldContext.small())) : Expressions.name(field); assertThat(name, is(fieldName)); } @@ -4118,7 +4119,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() { var value = Alias.unwrap(fields.get(0)); var math = as(value, Mod.class); assertThat(Expressions.name(math.left()), is("emp_no")); - assertThat(math.right().fold(), is(2)); + assertThat(math.right().fold(FoldContext.small()), is(2)); // languages + emp_no % 2 var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class); if (add.left() instanceof Mod mod) { @@ -4127,7 +4128,7 @@ public void testNestedExpressionsWithGroupingKeyInAggs() { assertThat(Expressions.name(add.left()), is("languages")); var mod = as(add.right().canonical(), Mod.class); assertThat(Expressions.name(mod.left()), is("emp_no")); - assertThat(mod.right().fold(), is(2)); + assertThat(mod.right().fold(FoldContext.small()), is(2)); } /** @@ -4156,7 +4157,7 @@ public void testNestedExpressionsWithMultiGrouping() { var value = Alias.unwrap(fields.get(0).canonical()); var math = as(value, Mod.class); assertThat(Expressions.name(math.left()), is("emp_no")); - assertThat(math.right().fold(), is(2)); + assertThat(math.right().fold(FoldContext.small()), is(2)); // languages + salary var add = as(Alias.unwrap(fields.get(1).canonical()), Add.class); assertThat(Expressions.name(add.left()), anyOf(is("languages"), is("salary"))); @@ -4173,7 +4174,7 @@ public void testNestedExpressionsWithMultiGrouping() { assertThat(Expressions.name(add3.right()), anyOf(is("salary"), is("languages"))); // emp_no % 2 assertThat(Expressions.name(mod.left()), is("emp_no")); - assertThat(mod.right().fold(), is(2)); + assertThat(mod.right().fold(FoldContext.small()), is(2)); } /** @@ -4611,8 +4612,8 @@ public void testCountOfLiteral() { var mvCoalesce = as(mul.left(), Coalesce.class); assertThat(mvCoalesce.children().size(), equalTo(2)); var mvCount = as(mvCoalesce.children().get(0), MvCount.class); - assertThat(mvCount.fold(), equalTo(2)); - assertThat(mvCoalesce.children().get(1).fold(), equalTo(0)); + assertThat(mvCount.fold(FoldContext.small()), equalTo(2)); + assertThat(mvCoalesce.children().get(1).fold(FoldContext.small()), equalTo(0)); var count = as(mul.right(), ReferenceAttribute.class); assertThat(count.name(), equalTo("$$COUNT$s$0")); @@ -4623,8 +4624,8 @@ public void testCountOfLiteral() { var mvCoalesce_expr = as(mul_expr.left(), Coalesce.class); assertThat(mvCoalesce_expr.children().size(), equalTo(2)); var mvCount_expr = as(mvCoalesce_expr.children().get(0), MvCount.class); - assertThat(mvCount_expr.fold(), equalTo(1)); - assertThat(mvCoalesce_expr.children().get(1).fold(), equalTo(0)); + assertThat(mvCount_expr.fold(FoldContext.small()), equalTo(1)); + assertThat(mvCoalesce_expr.children().get(1).fold(FoldContext.small()), equalTo(0)); var count_expr = as(mul_expr.right(), ReferenceAttribute.class); assertThat(count_expr.name(), equalTo("$$COUNT$s$0")); @@ -4636,7 +4637,7 @@ public void testCountOfLiteral() { assertThat(mvCoalesce_null.children().size(), equalTo(2)); var mvCount_null = as(mvCoalesce_null.children().get(0), MvCount.class); assertThat(mvCount_null.field(), equalTo(NULL)); - assertThat(mvCoalesce_null.children().get(1).fold(), equalTo(0)); + assertThat(mvCoalesce_null.children().get(1).fold(FoldContext.small()), equalTo(0)); var count_null = as(mul_null.right(), ReferenceAttribute.class); assertThat(count_null.name(), equalTo("$$COUNT$s$0")); } @@ -4675,7 +4676,7 @@ public void testSumOfLiteral() { assertThat(s.name(), equalTo("s")); var mul = as(s.child(), Mul.class); var mvSum = as(mul.left(), MvSum.class); - assertThat(mvSum.fold(), equalTo(3)); + assertThat(mvSum.fold(FoldContext.small()), equalTo(3)); var count = as(mul.right(), ReferenceAttribute.class); assertThat(count.name(), equalTo("$$COUNT$s$0")); @@ -4684,7 +4685,7 @@ public void testSumOfLiteral() { assertThat(s_expr.name(), equalTo("s_expr")); var mul_expr = as(s_expr.child(), Mul.class); var mvSum_expr = as(mul_expr.left(), MvSum.class); - assertThat(mvSum_expr.fold(), equalTo(3.14)); + assertThat(mvSum_expr.fold(FoldContext.small()), equalTo(3.14)); var count_expr = as(mul_expr.right(), ReferenceAttribute.class); assertThat(count_expr.name(), equalTo("$$COUNT$s$0")); @@ -4833,7 +4834,7 @@ private static void assertAggOfConstExprs(AggOfLiteralTestCase testCase, List x.foldable() ? new Literal(x.source(), x.fold(), x.dataType()) : x); + be -> LITERALS_ON_THE_RIGHT.rule(be, logicalOptimizerCtx) + ).transformUp(x -> x.foldable() ? new Literal(x.source(), x.fold(FoldContext.small()), x.dataType()) : x); List resolvedFields = fieldAttributeExp.collectFirstChildren(x -> x instanceof FieldAttribute); for (Expression field : resolvedFields) { @@ -5679,8 +5680,7 @@ public void testSimplifyComparisonArithmeticWithFloatsAndDirectionChange() { } private void assertNullLiteral(Expression expression) { - assertEquals(Literal.class, expression.getClass()); - assertNull(expression.fold()); + assertNull(as(expression, Literal.class).value()); } @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108519") @@ -5776,7 +5776,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsEquals() { var filter = as(limit.child(), Filter.class); var insensitive = as(filter.condition(), InsensitiveEquals.class); as(insensitive.left(), FieldAttribute.class); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is(value)); as(filter.child(), EsRelation.class); } @@ -5792,7 +5792,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsNotEquals() { var not = as(filter.condition(), Not.class); var insensitive = as(not.field(), InsensitiveEquals.class); as(insensitive.left(), FieldAttribute.class); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is(value)); as(filter.child(), EsRelation.class); } @@ -5805,7 +5805,7 @@ public void testReplaceStringCasingWithInsensitiveEqualsUnwrap() { var insensitive = as(filter.condition(), InsensitiveEquals.class); var field = as(insensitive.left(), FieldAttribute.class); assertThat(field.fieldName(), is("first_name")); - var bRef = as(insensitive.right().fold(), BytesRef.class); + var bRef = as(insensitive.right().fold(FoldContext.small()), BytesRef.class); assertThat(bRef.utf8ToString(), is("VALÜ")); as(filter.child(), EsRelation.class); } @@ -5856,7 +5856,7 @@ public void testLookupSimple() { var left = as(join.left(), EsqlProject.class); assertThat(left.output().toString(), containsString("int{r}")); var limit = as(left.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); assertThat(join.config().matchFields().stream().map(Object::toString).toList(), matchesList().item(startsWith("int{r}"))); @@ -5925,7 +5925,7 @@ public void testLookupStats() { } var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var agg = as(limit.child(), Aggregate.class); assertMap( @@ -6017,7 +6017,7 @@ public void testLookupJoinPushDownFilterOnJoinKeyWithRename() { assertThat(join.config().type(), equalTo(JoinTypes.LEFT)); var project = as(join.left(), Project.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); // assert that the rename has been undone var op = as(filter.condition(), GreaterThan.class); @@ -6061,7 +6061,7 @@ public void testLookupJoinPushDownFilterOnLeftSideField() { var project = as(join.left(), Project.class); var limit = as(project.child(), Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var op = as(filter.condition(), GreaterThan.class); var field = as(op.left(), FieldAttribute.class); @@ -6100,7 +6100,7 @@ public void testLookupJoinPushDownDisabledForLookupField() { var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var op = as(filter.condition(), Equals.class); @@ -6144,7 +6144,7 @@ public void testLookupJoinPushDownSeparatedForConjunctionBetweenLeftAndRightFiel var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); // filter kept in place, working on the right side var filter = as(limit.child(), Filter.class); EsqlBinaryComparison op = as(filter.condition(), Equals.class); @@ -6195,7 +6195,7 @@ public void testLookupJoinPushDownDisabledForDisjunctionBetweenLeftAndRightField var plan = optimizedPlan(query); var limit = as(plan, Limit.class); - assertThat(limit.limit().fold(), equalTo(1000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(1000)); var filter = as(limit.child(), Filter.class); var or = as(filter.condition(), Or.class); @@ -6289,7 +6289,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { as(addEval.child(), EsRelation.class); assertThat(Expressions.attribute(mul.left()).id(), equalTo(finalAggs.aggregates().get(1).id())); - assertThat(mul.right().fold(), equalTo(1.1)); + assertThat(mul.right().fold(FoldContext.small()), equalTo(1.1)); assertThat(finalAggs.aggregateType(), equalTo(Aggregate.AggregateType.STANDARD)); Max maxRate = as(Alias.unwrap(finalAggs.aggregates().get(0)), Max.class); @@ -6304,7 +6304,7 @@ public void testTranslateMixedAggsWithMathWithoutGrouping() { ToPartial toPartialMaxCost = as(Alias.unwrap(aggsByTsid.aggregates().get(1)), ToPartial.class); assertThat(Expressions.attribute(toPartialMaxCost.field()).id(), equalTo(addEval.fields().get(0).id())); assertThat(Expressions.attribute(add.left()).name(), equalTo("network.cost")); - assertThat(add.right().fold(), equalTo(0.2)); + assertThat(add.right().fold(FoldContext.small()), equalTo(0.2)); } public void testTranslateMetricsGroupedByOneDimension() { @@ -6533,7 +6533,7 @@ METRICS k8s avg(round(1.05 * rate(network.total_bytes_in))) BY bucket(@timestamp assertThat(Expressions.attribute(finalAgg.groupings().get(1)).id(), equalTo(aggsByTsid.aggregates().get(1).id())); assertThat(Expressions.attribute(mul.left()).id(), equalTo(aggsByTsid.aggregates().get(0).id())); - assertThat(mul.right().fold(), equalTo(1.05)); + assertThat(mul.right().fold(FoldContext.small()), equalTo(1.05)); assertThat(aggsByTsid.aggregateType(), equalTo(Aggregate.AggregateType.METRICS)); Rate rate = as(Alias.unwrap(aggsByTsid.aggregates().get(0)), Rate.class); assertThat(Expressions.attribute(rate.field()).name(), equalTo("network.total_bytes_in")); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java index ff710a90e8154..66891210a1e47 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PhysicalPlanOptimizerTests.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -164,6 +165,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.statsForMissingField; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyze; @@ -245,7 +247,7 @@ public PhysicalPlanOptimizerTests(String name, Configuration config) { @Before public void init() { parser = new EsqlParser(); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(config)); EsqlFunctionRegistry functionRegistry = new EsqlFunctionRegistry(); mapper = new Mapper(); @@ -1117,7 +1119,7 @@ public void testLimit() { var fieldExtract = as(project.child(), FieldExtractExec.class); var source = source(fieldExtract.child()); assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES)); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); } /** @@ -1199,7 +1201,7 @@ public void testPushLimitToSource() { var leaves = extract.collectLeaves(); assertEquals(1, leaves.size()); var source = as(leaves.get(0), EsQueryExec.class); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); // extra ints for doc id and emp_no_10 assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); } @@ -1246,7 +1248,7 @@ public void testPushLimitAndFilterToSource() { var source = source(extract.child()); assertThat(source.estimatedRowSize(), equalTo(allFieldRowSize + Integer.BYTES * 2)); - assertThat(source.limit().fold(), is(10)); + assertThat(source.limit().fold(FoldContext.small()), is(10)); var rq = as(sv(source.query(), "emp_no"), RangeQueryBuilder.class); assertThat(rq.fieldName(), equalTo("emp_no")); assertThat(rq.from(), equalTo(0)); @@ -2902,14 +2904,14 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() { var eval = as(project.child(), EvalExec.class); var limit = as(eval.child(), LimitExec.class); assertThat(limit.limit(), instanceOf(Literal.class)); - assertThat(limit.limit().fold(), equalTo(10000)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(10000)); var aggFinal = as(limit.child(), AggregateExec.class); assertThat(aggFinal.getMode(), equalTo(FINAL)); var aggPartial = as(aggFinal.child(), AggregateExec.class); assertThat(aggPartial.getMode(), equalTo(INITIAL)); limit = as(aggPartial.child(), LimitExec.class); assertThat(limit.limit(), instanceOf(Literal.class)); - assertThat(limit.limit().fold(), equalTo(10)); + assertThat(limit.limit().fold(FoldContext.small()), equalTo(10)); var exchange = as(limit.child(), ExchangeExec.class); project = as(exchange.child(), ProjectExec.class); @@ -2918,7 +2920,7 @@ public void testAvgSurrogateFunctionAfterRenameAndLimit() { var fieldExtract = as(project.child(), FieldExtractExec.class); assertThat(Expressions.names(fieldExtract.attributesToExtract()), is(expectedFields)); var source = source(fieldExtract.child()); - assertThat(source.limit().fold(), equalTo(10)); + assertThat(source.limit().fold(FoldContext.small()), equalTo(10)); } /** @@ -4849,15 +4851,15 @@ public void testPushSpatialDistanceMultiEvalToSource() { var alias1 = as(eval.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(eval.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); // Validate the filter condition var and = as(filter.condition(), And.class); @@ -6205,15 +6207,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndCompoundEvalToSourc var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(evalExec.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); extract = as(evalExec.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("location")); var source = source(extract.child()); @@ -6294,7 +6296,7 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() { var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias4 = as(evalExec.fields().get(3), Alias.class); assertThat(alias4.name(), is("loc2")); as(alias4.child(), FieldAttribute.class); @@ -6307,8 +6309,8 @@ public void testPushCompoundTopNDistanceWithDeeplyNestedCompoundEvalToSource() { var refLocation = as(stDistance.left(), ReferenceAttribute.class); assertThat(refLocation.name(), is("loc3")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); var alias7 = as(evalExec.fields().get(6), Alias.class); assertThat(alias7.name(), is("distance")); as(alias7.child(), ReferenceAttribute.class); @@ -6391,15 +6393,15 @@ public void testPushCompoundTopNDistanceWithCompoundFilterAndNestedCompoundEvalT var alias1 = as(evalExec.fields().get(0), Alias.class); assertThat(alias1.name(), is("poi")); var poi = as(alias1.child(), Literal.class); - assertThat(poi.fold(), instanceOf(BytesRef.class)); + assertThat(poi.value(), instanceOf(BytesRef.class)); var alias2 = as(evalExec.fields().get(1), Alias.class); assertThat(alias2.name(), is("distance")); var stDistance = as(alias2.child(), StDistance.class); var location = as(stDistance.left(), FieldAttribute.class); assertThat(location.fieldName(), is("location")); var poiRef = as(stDistance.right(), Literal.class); - assertThat(poiRef.fold(), instanceOf(BytesRef.class)); - assertThat(poiRef.fold().toString(), is(poi.fold().toString())); + assertThat(poiRef.value(), instanceOf(BytesRef.class)); + assertThat(poiRef.value().toString(), is(poi.value().toString())); extract = as(evalExec.child(), FieldExtractExec.class); assertThat(names(extract.attributesToExtract()), contains("location")); var source = source(extract.child()); @@ -6931,7 +6933,7 @@ public void testManyEnrich() { var fragment = as(exchange.child(), FragmentExec.class); var partialTopN = as(fragment.fragment(), TopN.class); var enrich2 = as(partialTopN.child(), Enrich.class); - assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments")); + assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments")); assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY)); var eval = as(enrich2.child(), Eval.class); as(eval.child(), EsRelation.class); @@ -6957,7 +6959,7 @@ public void testManyEnrich() { var fragment = as(exchange.child(), FragmentExec.class); var partialTopN = as(fragment.fragment(), TopN.class); var enrich2 = as(partialTopN.child(), Enrich.class); - assertThat(BytesRefs.toString(enrich2.policyName().fold()), equalTo("departments")); + assertThat(BytesRefs.toString(enrich2.policyName().fold(FoldContext.small())), equalTo("departments")); assertThat(enrich2.mode(), equalTo(Enrich.Mode.ANY)); var eval = as(enrich2.child(), Eval.class); as(eval.child(), EsRelation.class); @@ -7549,7 +7551,7 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP // The TopN needs an estimated row size for the planner to work var plans = PlannerUtils.breakPlanBetweenCoordinatorAndDataNode(EstimatesRowSize.estimateRowSize(0, plan), config); plan = useDataNodePlan ? plans.v2() : plans.v1(); - plan = PlannerUtils.localPlan(List.of(), config, plan); + plan = PlannerUtils.localPlan(List.of(), config, FoldContext.small(), plan); LocalExecutionPlanner planner = new LocalExecutionPlanner( "test", "", @@ -7562,10 +7564,10 @@ private LocalExecutionPlanner.LocalExecutionPlan physicalOperationsFromPhysicalP new ExchangeSinkHandler(null, 10, () -> 10), null, null, - new EsPhysicalOperationProviders(List.of(), null) + new EsPhysicalOperationProviders(FoldContext.small(), List.of(), null) ); - return planner.plan(plan); + return planner.plan(FoldContext.small(), plan); } private List> findFieldNamesInLookupJoinDescription(LocalExecutionPlanner.LocalExecutionPlan physicalOperations) { @@ -7662,7 +7664,7 @@ public void testReductionPlanForTopN() { PhysicalPlan reduction = PlannerUtils.reductionPlan(plans.v2()); TopNExec reductionTopN = as(reduction, TopNExec.class); assertThat(reductionTopN.estimatedRowSize(), equalTo(allFieldRowSize)); - assertThat(reductionTopN.limit().fold(), equalTo(limit)); + assertThat(reductionTopN.limit().fold(FoldContext.small()), equalTo(limit)); } public void testReductionPlanForAggs() { @@ -7818,7 +7820,7 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // individually hence why here the plan is kept as is var l = p.transformUp(FragmentExec.class, fragment -> { - var localPlan = PlannerUtils.localPlan(config, fragment, searchStats); + var localPlan = PlannerUtils.localPlan(config, FoldContext.small(), fragment, searchStats); return EstimatesRowSize.estimateRowSize(fragment.estimatedRowSize(), localPlan); }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java index 9fe479dbb8625..e6a7d110f8c09 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/TestPlannerOptimizer.java @@ -9,6 +9,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; @@ -26,7 +27,7 @@ public class TestPlannerOptimizer { private final Configuration config; public TestPlannerOptimizer(Configuration config, Analyzer analyzer) { - this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config))); + this(config, analyzer, new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()))); } public TestPlannerOptimizer(Configuration config, Analyzer analyzer, LogicalPlanOptimizer logicalOptimizer) { @@ -61,8 +62,13 @@ private PhysicalPlan optimizedPlan(PhysicalPlan plan, SearchStats searchStats) { // this is of no use in the unit tests, which checks the plan as a whole instead of each // individually hence why here the plan is kept as is - var logicalTestOptimizer = new LocalLogicalPlanOptimizer(new LocalLogicalOptimizerContext(config, searchStats)); - var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer(new LocalPhysicalOptimizerContext(config, searchStats), true); + var logicalTestOptimizer = new LocalLogicalPlanOptimizer( + new LocalLogicalOptimizerContext(config, FoldContext.small(), searchStats) + ); + var physicalTestOptimizer = new TestLocalPhysicalPlanOptimizer( + new LocalPhysicalOptimizerContext(config, FoldContext.small(), searchStats), + true + ); var l = PlannerUtils.localPlan(physicalPlan, logicalTestOptimizer, physicalTestOptimizer); // handle local reduction alignment diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java new file mode 100644 index 0000000000000..5d2fec0fc8181 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/LogicalOptimizerContextTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; +import org.elasticsearch.xpack.esql.ConfigurationTestUtils; +import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.session.Configuration; + +import static org.hamcrest.Matchers.equalTo; + +public class LogicalOptimizerContextTests extends ESTestCase { + public void testToString() { + // Random looking numbers for FoldContext are indeed random. Just so we have consistent numbers to assert on in toString. + LogicalOptimizerContext ctx = new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG, new FoldContext(102)); + ctx.foldCtx().trackAllocation(Source.EMPTY, 99); + assertThat( + ctx.toString(), + equalTo("LogicalOptimizerContext[configuration=" + EsqlTestUtils.TEST_CFG + ", foldCtx=FoldContext[3/102]]") + ); + } + + public void testEqualsAndHashCode() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode(randomLogicalOptimizerContext(), this::copy, this::mutate); + } + + private LogicalOptimizerContext randomLogicalOptimizerContext() { + return new LogicalOptimizerContext(ConfigurationTestUtils.randomConfiguration(), randomFoldContext()); + } + + private LogicalOptimizerContext copy(LogicalOptimizerContext c) { + return new LogicalOptimizerContext(c.configuration(), c.foldCtx()); + } + + private LogicalOptimizerContext mutate(LogicalOptimizerContext c) { + Configuration configuration = c.configuration(); + FoldContext foldCtx = c.foldCtx(); + if (randomBoolean()) { + configuration = randomValueOtherThan(configuration, ConfigurationTestUtils::randomConfiguration); + } else { + foldCtx = randomValueOtherThan(foldCtx, this::randomFoldContext); + } + return new LogicalOptimizerContext(configuration, foldCtx); + } + + private FoldContext randomFoldContext() { + FoldContext ctx = new FoldContext(randomNonNegativeLong()); + if (randomBoolean()) { + ctx.trackAllocation(Source.EMPTY, randomLongBetween(0, ctx.initialAllowedBytes())); + } + return ctx; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java index 08c8612d8097c..c0c145aee5382 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanFunctionEqualsEliminationTests.java @@ -22,25 +22,26 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class BooleanFunctionEqualsEliminationTests extends ESTestCase { + private Expression booleanFunctionEqualElimination(BinaryComparison e) { + return new BooleanFunctionEqualsElimination().rule(e, unboundLogicalOptimizerContext()); + } public void testBoolEqualsSimplificationOnExpressions() { - BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination(); Expression exp = new GreaterThan(EMPTY, getFieldAttribute(), new Literal(EMPTY, 0, DataType.INTEGER), null); - assertEquals(exp, s.rule(new Equals(EMPTY, exp, TRUE))); + assertEquals(exp, booleanFunctionEqualElimination(new Equals(EMPTY, exp, TRUE))); // TODO: Replace use of QL Not with ESQL Not - assertEquals(new Not(EMPTY, exp), s.rule(new Equals(EMPTY, exp, FALSE))); + assertEquals(new Not(EMPTY, exp), booleanFunctionEqualElimination(new Equals(EMPTY, exp, FALSE))); } public void testBoolEqualsSimplificationOnFields() { - BooleanFunctionEqualsElimination s = new BooleanFunctionEqualsElimination(); - FieldAttribute field = getFieldAttribute(); List comparisons = asList( @@ -55,7 +56,7 @@ public void testBoolEqualsSimplificationOnFields() { ); for (BinaryComparison comparison : comparisons) { - assertEquals(comparison, s.rule(comparison)); + assertEquals(comparison, booleanFunctionEqualElimination(comparison)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java index 3b1f8cfc83af3..5b4bf806518de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/BooleanSimplificationTests.java @@ -9,9 +9,11 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -20,33 +22,31 @@ public class BooleanSimplificationTests extends ESTestCase { private static final Expression DUMMY_EXPRESSION = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0); - public void testBoolSimplifyOr() { - BooleanSimplification simplification = new BooleanSimplification(); + private Expression booleanSimplification(ScalarFunction e) { + return new BooleanSimplification().rule(e, unboundLogicalOptimizerContext()); + } - assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, TRUE))); - assertEquals(TRUE, simplification.rule(new Or(EMPTY, TRUE, DUMMY_EXPRESSION))); - assertEquals(TRUE, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, TRUE))); + public void testBoolSimplifyOr() { + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, TRUE))); + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, TRUE, DUMMY_EXPRESSION))); + assertEquals(TRUE, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, TRUE))); - assertEquals(FALSE, simplification.rule(new Or(EMPTY, FALSE, FALSE))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, FALSE, DUMMY_EXPRESSION))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new Or(EMPTY, DUMMY_EXPRESSION, FALSE))); + assertEquals(FALSE, booleanSimplification(new Or(EMPTY, FALSE, FALSE))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, FALSE, DUMMY_EXPRESSION))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new Or(EMPTY, DUMMY_EXPRESSION, FALSE))); } public void testBoolSimplifyAnd() { - BooleanSimplification simplification = new BooleanSimplification(); + assertEquals(TRUE, booleanSimplification(new And(EMPTY, TRUE, TRUE))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, TRUE, DUMMY_EXPRESSION))); + assertEquals(DUMMY_EXPRESSION, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, TRUE))); - assertEquals(TRUE, simplification.rule(new And(EMPTY, TRUE, TRUE))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, TRUE, DUMMY_EXPRESSION))); - assertEquals(DUMMY_EXPRESSION, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, TRUE))); - - assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, FALSE))); - assertEquals(FALSE, simplification.rule(new And(EMPTY, FALSE, DUMMY_EXPRESSION))); - assertEquals(FALSE, simplification.rule(new And(EMPTY, DUMMY_EXPRESSION, FALSE))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, FALSE))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, FALSE, DUMMY_EXPRESSION))); + assertEquals(FALSE, booleanSimplification(new And(EMPTY, DUMMY_EXPRESSION, FALSE))); } public void testBoolCommonFactorExtraction() { - BooleanSimplification simplification = new BooleanSimplification(); - Expression a1 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); Expression a2 = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 1); Expression b = new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 2); @@ -55,7 +55,7 @@ public void testBoolCommonFactorExtraction() { Or actual = new Or(EMPTY, new And(EMPTY, a1, b), new And(EMPTY, a2, c)); And expected = new And(EMPTY, a1, new Or(EMPTY, b, c)); - assertEquals(expected, simplification.rule(actual)); + assertEquals(expected, booleanSimplification(actual)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java index d388369e0b167..a0d23731ae82d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineBinaryComparisonsTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; @@ -37,6 +38,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -45,19 +47,17 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; public class CombineBinaryComparisonsTests extends ESTestCase { - - private static final Expression DUMMY_EXPRESSION = - new org.elasticsearch.xpack.esql.core.optimizer.OptimizerRulesTests.DummyBooleanExpression(EMPTY, 0); + private Expression combine(BinaryLogic e) { + return new CombineBinaryComparisons().rule(e, unboundLogicalOptimizerContext()); + } public void testCombineBinaryComparisonsNotComparable() { FieldAttribute fa = getFieldAttribute(); LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX); LessThan lt = lessThanOf(fa, FALSE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - And and = new And(EMPTY, lte, lt); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(exp, and); } @@ -67,9 +67,7 @@ public void testCombineBinaryComparisonsUpper() { LessThanOrEqual lte = lessThanOrEqualOf(fa, SIX); LessThan lt = lessThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, lte, lt)); + Expression exp = combine(new And(EMPTY, lte, lt)); assertEquals(LessThan.class, exp.getClass()); LessThan r = (LessThan) exp; assertEquals(FIVE, r.right()); @@ -81,9 +79,7 @@ public void testCombineBinaryComparisonsLower() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, SIX); GreaterThan gt = greaterThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, gt)); + Expression exp = combine(new And(EMPTY, gte, gt)); assertEquals(GreaterThanOrEqual.class, exp.getClass()); GreaterThanOrEqual r = (GreaterThanOrEqual) exp; assertEquals(SIX, r.right()); @@ -95,9 +91,7 @@ public void testCombineBinaryComparisonsInclude() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, FIVE); GreaterThan gt = greaterThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, gt)); + Expression exp = combine(new And(EMPTY, gte, gt)); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan r = (GreaterThan) exp; assertEquals(FIVE, r.right()); @@ -111,9 +105,7 @@ public void testCombineMultipleBinaryComparisons() { LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7)); LessThan lt = lessThanOf(fa, SIX); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - - Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte)))); + Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, gt, new And(EMPTY, lt, lte)))); assertEquals(And.class, exp.getClass()); And and = (And) exp; assertEquals(gt, and.left()); @@ -128,10 +120,8 @@ public void testCombineMixedMultipleBinaryComparisons() { LessThanOrEqual lte = lessThanOrEqualOf(fa, L(7)); Expression ne = notEqualsOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - // TRUE AND a != 5 AND 4 < a <= 7 - Expression exp = rule.rule(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte))))); + Expression exp = combine(new And(EMPTY, gte, new And(EMPTY, TRUE, new And(EMPTY, gt, new And(EMPTY, ne, lte))))); assertEquals(And.class, exp.getClass()); And and = ((And) exp); assertEquals(And.class, and.right().getClass()); @@ -150,8 +140,7 @@ public void testCombineComparisonsIntoRange() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE); LessThan lt = lessThanOf(fa, FIVE); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(new And(EMPTY, gte, lt)); + Expression exp = combine(new And(EMPTY, gte, lt)); assertEquals(And.class, exp.getClass()); And and = (And) exp; @@ -167,8 +156,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGt3() { GreaterThan gt = greaterThanOf(fa, THREE); And and = new And(EMPTY, neq, gt); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(gt, exp); } @@ -180,8 +168,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte2() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO); And and = new And(EMPTY, neq, gte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; assertEquals(TWO, gt.right()); @@ -195,8 +182,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndGte1() { GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, ONE); And and = new And(EMPTY, neq, gte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(And.class, exp.getClass()); // can't optimize } @@ -208,8 +194,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte3() { LessThanOrEqual lte = lessThanOrEqualOf(fa, THREE); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(and, exp); // can't optimize } @@ -221,8 +206,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte2() { LessThanOrEqual lte = lessThanOrEqualOf(fa, TWO); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(LessThan.class, exp.getClass()); LessThan lt = (LessThan) exp; assertEquals(TWO, lt.right()); @@ -236,8 +220,7 @@ public void testCombineBinaryComparisonsConjunction_Neq2AndLte1() { LessThanOrEqual lte = lessThanOrEqualOf(fa, ONE); And and = new And(EMPTY, neq, lte); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals(lte, exp); } @@ -251,8 +234,7 @@ public void testCombineBinaryComparisonsDisjunctionNotComparable() { Or or = new Or(EMPTY, gt1, gt2); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(exp, or); } @@ -266,8 +248,7 @@ public void testCombineBinaryComparisonsDisjunctionLowerBound() { Or or = new Or(EMPTY, gt1, new Or(EMPTY, gt2, gt3)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; @@ -284,8 +265,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeLowerBounds() { Or or = new Or(EMPTY, new Or(EMPTY, gt1, gt2), gte3); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(GreaterThan.class, exp.getClass()); GreaterThan gt = (GreaterThan) exp; @@ -302,8 +282,7 @@ public void testCombineBinaryComparisonsDisjunctionUpperBound() { Or or = new Or(EMPTY, new Or(EMPTY, lt1, lt2), lt3); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(LessThan.class, exp.getClass()); LessThan lt = (LessThan) exp; @@ -320,8 +299,7 @@ public void testCombineBinaryComparisonsDisjunctionIncludeUpperBounds() { Or or = new Or(EMPTY, lt2, new Or(EMPTY, lte2, lt1)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(LessThanOrEqual.class, exp.getClass()); LessThanOrEqual lte = (LessThanOrEqual) exp; @@ -340,8 +318,7 @@ public void testCombineBinaryComparisonsDisjunctionOfLowerAndUpperBounds() { Or or = new Or(EMPTY, new Or(EMPTY, lt2, gt3), new Or(EMPTY, lt1, gt4)); - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(or); + Expression exp = combine(or); assertEquals(Or.class, exp.getClass()); Or ro = (Or) exp; @@ -367,7 +344,7 @@ public void testBooleanSimplificationCommonExpressionSubstraction() { And right = new And(EMPTY, a2, common); Or or = new Or(EMPTY, left, right); - Expression exp = new BooleanSimplification().rule(or); + Expression exp = new BooleanSimplification().rule(or, unboundLogicalOptimizerContext()); assertEquals(new And(EMPTY, common, new Or(EMPTY, a1, a2)), exp); } @@ -391,8 +368,7 @@ public void testBinaryComparisonAndOutOfRangeNotEqualsDifferentFields() { ); for (And and : testCases) { - CombineBinaryComparisons rule = new CombineBinaryComparisons(); - Expression exp = rule.rule(and); + Expression exp = combine(and); assertEquals("Rule should not have transformed [" + and.nodeString() + "]", and, exp); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java index 043d18dac9fd4..bb5f2fd3505e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/CombineDisjunctionsTests.java @@ -38,16 +38,24 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.randomLiteral; import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.hamcrest.Matchers.contains; public class CombineDisjunctionsTests extends ESTestCase { + private Expression combineDisjunctions(Or e) { + return new CombineDisjunctions().rule(e, unboundLogicalOptimizerContext()); + } + + private LogicalPlan combineDisjunctions(LogicalPlan l) { + return new CombineDisjunctions().apply(l, unboundLogicalOptimizerContext()); + } public void testTwoEqualsWithOr() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -58,7 +66,7 @@ public void testTwoEqualsWithSameValue() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), equalsOf(fa, ONE)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); @@ -69,7 +77,7 @@ public void testOneEqualsOneIn() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, List.of(TWO))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -80,7 +88,7 @@ public void testOneEqualsOneInWithSameValue() { FieldAttribute fa = getFieldAttribute(); Or or = new Or(EMPTY, equalsOf(fa, ONE), new In(EMPTY, fa, asList(ONE, TWO))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -92,7 +100,7 @@ public void testSingleValueInToEquals() { Equals equals = equalsOf(fa, ONE); Or or = new Or(EMPTY, equals, new In(EMPTY, fa, List.of(ONE))); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(equals, e); } @@ -101,7 +109,7 @@ public void testEqualsBehindAnd() { And and = new And(EMPTY, equalsOf(fa, ONE), equalsOf(fa, TWO)); Filter dummy = new Filter(EMPTY, relation(), and); - LogicalPlan transformed = new CombineDisjunctions().apply(dummy); + LogicalPlan transformed = combineDisjunctions(dummy); assertSame(dummy, transformed); assertEquals(and, ((Filter) transformed).condition()); } @@ -111,7 +119,7 @@ public void testTwoEqualsDifferentFields() { FieldAttribute fieldTwo = getFieldAttribute("TWO"); Or or = new Or(EMPTY, equalsOf(fieldOne, ONE), equalsOf(fieldTwo, TWO)); - Expression e = new CombineDisjunctions().rule(or); + Expression e = combineDisjunctions(or); assertEquals(or, e); } @@ -120,7 +128,7 @@ public void testMultipleIn() { Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), new In(EMPTY, fa, List.of(TWO))); Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctions().rule(secondOr); + Expression e = combineDisjunctions(secondOr); assertEquals(In.class, e.getClass()); In in = (In) e; assertEquals(fa, in.value()); @@ -132,7 +140,7 @@ public void testOrWithNonCombinableExpressions() { Or firstOr = new Or(EMPTY, new In(EMPTY, fa, List.of(ONE)), lessThanOf(fa, TWO)); Or secondOr = new Or(EMPTY, firstOr, new In(EMPTY, fa, List.of(THREE))); - Expression e = new CombineDisjunctions().rule(secondOr); + Expression e = combineDisjunctions(secondOr); assertEquals(Or.class, e.getClass()); Or or = (Or) e; assertEquals(or.left(), firstOr.right()); @@ -160,7 +168,7 @@ public void testCombineCIDRMatch() { cidrs.add(new CIDRMatch(EMPTY, faa, ipa2)); cidrs.add(new CIDRMatch(EMPTY, fab, ipb2)); Or oldOr = (Or) Predicates.combineOr(cidrs); - Expression e = new CombineDisjunctions().rule(oldOr); + Expression e = combineDisjunctions(oldOr); assertEquals(Or.class, e.getClass()); Or newOr = (Or) e; assertEquals(CIDRMatch.class, newOr.left().getClass()); @@ -211,7 +219,7 @@ public void testCombineCIDRMatchEqualsIns() { Or oldOr = (Or) Predicates.combineOr(all); - Expression e = new CombineDisjunctions().rule(oldOr); + Expression e = combineDisjunctions(oldOr); assertEquals(Or.class, e.getClass()); Or newOr = (Or) e; assertEquals(Or.class, newOr.left().getClass()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java index 01af91271e1ba..8a8585b8d0ab5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ConstantFoldingTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.predicate.BinaryOperator; @@ -45,68 +46,77 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.of; import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class ConstantFoldingTests extends ESTestCase { + private Expression constantFolding(Expression e) { + return new ConstantFolding().rule(e, unboundLogicalOptimizerContext()); + } public void testConstantFolding() { Expression exp = new Add(EMPTY, TWO, THREE); assertTrue(exp.foldable()); - Expression result = new ConstantFolding().rule(exp); - assertTrue(result instanceof Literal); - assertEquals(5, ((Literal) result).value()); + Expression result = constantFolding(exp); + assertEquals(5, as(result, Literal.class).value()); // check now with an alias - result = new ConstantFolding().rule(new Alias(EMPTY, "a", exp)); + result = constantFolding(new Alias(EMPTY, "a", exp)); assertEquals("a", Expressions.name(result)); assertEquals(Alias.class, result.getClass()); } public void testConstantFoldingBinaryComparison() { - assertEquals(FALSE, new ConstantFolding().rule(greaterThanOf(TWO, THREE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(greaterThanOrEqualOf(TWO, THREE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(equalsOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(notEqualsOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(lessThanOrEqualOf(TWO, THREE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(lessThanOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(greaterThanOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(greaterThanOrEqualOf(TWO, THREE)).canonical()); + assertEquals(FALSE, constantFolding(equalsOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(notEqualsOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(lessThanOrEqualOf(TWO, THREE)).canonical()); + assertEquals(TRUE, constantFolding(lessThanOf(TWO, THREE)).canonical()); } public void testConstantFoldingBinaryLogic() { - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical()); + assertEquals(FALSE, constantFolding(new And(EMPTY, greaterThanOf(TWO, THREE), TRUE)).canonical()); + assertEquals(TRUE, constantFolding(new Or(EMPTY, greaterThanOrEqualOf(TWO, THREE), TRUE)).canonical()); } public void testConstantFoldingBinaryLogic_WithNullHandling() { - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, TRUE)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, TRUE, NULL)).canonical().nullable()); - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, NULL, FALSE)).canonical()); - assertEquals(FALSE, new ConstantFolding().rule(new And(EMPTY, FALSE, NULL)).canonical()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new And(EMPTY, NULL, NULL)).canonical().nullable()); - - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, TRUE)).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new Or(EMPTY, TRUE, NULL)).canonical()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, FALSE)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, FALSE, NULL)).canonical().nullable()); - assertEquals(Nullability.TRUE, new ConstantFolding().rule(new Or(EMPTY, NULL, NULL)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, TRUE)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, TRUE, NULL)).canonical().nullable()); + assertEquals(FALSE, constantFolding(new And(EMPTY, NULL, FALSE)).canonical()); + assertEquals(FALSE, constantFolding(new And(EMPTY, FALSE, NULL)).canonical()); + assertEquals(Nullability.TRUE, constantFolding(new And(EMPTY, NULL, NULL)).canonical().nullable()); + + assertEquals(TRUE, constantFolding(new Or(EMPTY, NULL, TRUE)).canonical()); + assertEquals(TRUE, constantFolding(new Or(EMPTY, TRUE, NULL)).canonical()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, FALSE)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, FALSE, NULL)).canonical().nullable()); + assertEquals(Nullability.TRUE, constantFolding(new Or(EMPTY, NULL, NULL)).canonical().nullable()); } public void testConstantFoldingRange() { - assertEquals(true, new ConstantFolding().rule(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold()); - assertEquals(false, new ConstantFolding().rule(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold()); + assertEquals( + true, + constantFolding(rangeOf(FIVE, FIVE, true, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small()) + ); + assertEquals( + false, + constantFolding(rangeOf(FIVE, FIVE, false, new Literal(EMPTY, 10, DataType.INTEGER), false)).fold(FoldContext.small()) + ); } public void testConstantNot() { - assertEquals(FALSE, new ConstantFolding().rule(new Not(EMPTY, TRUE))); - assertEquals(TRUE, new ConstantFolding().rule(new Not(EMPTY, FALSE))); + assertEquals(FALSE, constantFolding(new Not(EMPTY, TRUE))); + assertEquals(TRUE, constantFolding(new Not(EMPTY, FALSE))); } public void testConstantFoldingLikes() { - assertEquals(TRUE, new ConstantFolding().rule(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical()); - assertEquals(TRUE, new ConstantFolding().rule(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical()); + assertEquals(TRUE, constantFolding(new WildcardLike(EMPTY, of("test_emp"), new WildcardPattern("test*"))).canonical()); + assertEquals(TRUE, constantFolding(new RLike(EMPTY, of("test_emp"), new RLikePattern("test.emp"))).canonical()); } public void testArithmeticFolding() { @@ -125,7 +135,7 @@ public void testFoldRange() { Expression value = new Literal(EMPTY, 12, DataType.INTEGER); Range range = new Range(EMPTY, value, lowerBound, randomBoolean(), upperBound, randomBoolean(), randomZone()); - Expression folded = new ConstantFolding().rule(range); + Expression folded = constantFolding(range); assertTrue((Boolean) as(folded, Literal.class).value()); } @@ -156,16 +166,15 @@ public void testFoldRangeWithInvalidBoundaries() { // Just applying this to the range directly won't perform a transformDown. LogicalPlan filter = new Filter(EMPTY, emptySource(), range); - Filter foldedOnce = as(new ConstantFolding().apply(filter), Filter.class); + Filter foldedOnce = as(new ConstantFolding().apply(filter, unboundLogicalOptimizerContext()), Filter.class); // We need to run the rule twice, because during the first run only the boundaries can be folded - the range doesn't know it's // foldable, yet. - Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce), Filter.class); + Filter foldedTwice = as(new ConstantFolding().apply(foldedOnce, unboundLogicalOptimizerContext()), Filter.class); assertFalse((Boolean) as(foldedTwice.condition(), Literal.class).value()); } - private static Object foldOperator(BinaryOperator b) { - return ((Literal) new ConstantFolding().rule(b)).value(); + private Object foldOperator(BinaryOperator b) { + return ((Literal) constantFolding(b)).value(); } - } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java index ae31576184938..252b25a214bb8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/FoldNullTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; @@ -67,9 +68,11 @@ import java.util.List; import static org.elasticsearch.xpack.esql.EsqlTestUtils.L; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.configuration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.NULL; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -85,28 +88,27 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; public class FoldNullTests extends ESTestCase { + private Expression foldNull(Expression e) { + return new FoldNull().rule(e, unboundLogicalOptimizerContext()); + } public void testBasicNullFolding() { - FoldNull rule = new FoldNull(); - assertNullLiteral(rule.rule(new Add(EMPTY, L(randomInt()), Literal.NULL))); - assertNullLiteral(rule.rule(new Round(EMPTY, Literal.NULL, null))); - assertNullLiteral(rule.rule(new Pow(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null))); - assertNullLiteral(rule.rule(new DateParse(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL))); - assertNullLiteral(rule.rule(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new Add(EMPTY, L(randomInt()), Literal.NULL))); + assertNullLiteral(foldNull(new Round(EMPTY, Literal.NULL, null))); + assertNullLiteral(foldNull(new Pow(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new DateFormat(EMPTY, Literal.NULL, Literal.NULL, null))); + assertNullLiteral(foldNull(new DateParse(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new DateTrunc(EMPTY, Literal.NULL, Literal.NULL))); + assertNullLiteral(foldNull(new Substring(EMPTY, Literal.NULL, Literal.NULL, Literal.NULL))); } public void testNullFoldingIsNotNull() { - FoldNull foldNull = new FoldNull(); - assertEquals(true, foldNull.rule(new IsNotNull(EMPTY, TRUE)).fold()); - assertEquals(false, foldNull.rule(new IsNotNull(EMPTY, NULL)).fold()); + assertEquals(true, foldNull(new IsNotNull(EMPTY, TRUE)).fold(FoldContext.small())); + assertEquals(false, foldNull(new IsNotNull(EMPTY, NULL)).fold(FoldContext.small())); } @SuppressWarnings("unchecked") public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exception { - FoldNull rule = new FoldNull(); - List> items = List.of( MvDedupe.class, MvFirst.class, @@ -119,119 +121,112 @@ public void testNullFoldingDoesNotApplyOnAbstractMultivalueFunction() throws Exc for (Class clazz : items) { Constructor ctor = clazz.getConstructor(Source.class, Expression.class); AbstractMultivalueFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a")); - assertEquals(conditionalFunction, rule.rule(conditionalFunction)); + assertEquals(conditionalFunction, foldNull(conditionalFunction)); conditionalFunction = ctor.newInstance(EMPTY, NULL); - assertEquals(NULL, rule.rule(conditionalFunction)); + assertEquals(NULL, foldNull(conditionalFunction)); } // avg and count ar different just because they know the return type in advance (all the others infer the type from the input) MvAvg avg = new MvAvg(EMPTY, getFieldAttribute("a")); - assertEquals(avg, rule.rule(avg)); + assertEquals(avg, foldNull(avg)); avg = new MvAvg(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg)); MvCount count = new MvCount(EMPTY, getFieldAttribute("a")); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); count = new MvCount(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, INTEGER), rule.rule(count)); + assertEquals(new Literal(EMPTY, null, INTEGER), foldNull(count)); } public void testNullFoldingIsNull() { - FoldNull foldNull = new FoldNull(); - assertEquals(true, foldNull.rule(new IsNull(EMPTY, NULL)).fold()); - assertEquals(false, foldNull.rule(new IsNull(EMPTY, TRUE)).fold()); + assertEquals(true, foldNull(new IsNull(EMPTY, NULL)).fold(FoldContext.small())); + assertEquals(false, foldNull(new IsNull(EMPTY, TRUE)).fold(FoldContext.small())); } public void testGenericNullableExpression() { FoldNull rule = new FoldNull(); // arithmetic - assertNullLiteral(rule.rule(new Add(EMPTY, getFieldAttribute("a"), NULL))); + assertNullLiteral(foldNull(new Add(EMPTY, getFieldAttribute("a"), NULL))); // comparison - assertNullLiteral(rule.rule(greaterThanOf(getFieldAttribute("a"), NULL))); + assertNullLiteral(foldNull(greaterThanOf(getFieldAttribute("a"), NULL))); // regex - assertNullLiteral(rule.rule(new RLike(EMPTY, NULL, new RLikePattern("123")))); + assertNullLiteral(foldNull(new RLike(EMPTY, NULL, new RLikePattern("123")))); // date functions - assertNullLiteral(rule.rule(new DateExtract(EMPTY, NULL, NULL, configuration("")))); + assertNullLiteral(foldNull(new DateExtract(EMPTY, NULL, NULL, configuration("")))); // math functions - assertNullLiteral(rule.rule(new Cos(EMPTY, NULL))); + assertNullLiteral(foldNull(new Cos(EMPTY, NULL))); // string functions - assertNullLiteral(rule.rule(new LTrim(EMPTY, NULL))); + assertNullLiteral(foldNull(new LTrim(EMPTY, NULL))); // spatial - assertNullLiteral(rule.rule(new SpatialCentroid(EMPTY, NULL))); + assertNullLiteral(foldNull(new SpatialCentroid(EMPTY, NULL))); // ip - assertNullLiteral(rule.rule(new CIDRMatch(EMPTY, NULL, List.of(NULL)))); + assertNullLiteral(foldNull(new CIDRMatch(EMPTY, NULL, List.of(NULL)))); // conversion - assertNullLiteral(rule.rule(new ToString(EMPTY, NULL))); + assertNullLiteral(foldNull(new ToString(EMPTY, NULL))); } public void testNullFoldingDoesNotApplyOnLogicalExpressions() { - FoldNull rule = new FoldNull(); - Or or = new Or(EMPTY, NULL, TRUE); - assertEquals(or, rule.rule(or)); + assertEquals(or, foldNull(or)); or = new Or(EMPTY, NULL, NULL); - assertEquals(or, rule.rule(or)); + assertEquals(or, foldNull(or)); And and = new And(EMPTY, NULL, TRUE); - assertEquals(and, rule.rule(and)); + assertEquals(and, foldNull(and)); and = new And(EMPTY, NULL, NULL); - assertEquals(and, rule.rule(and)); + assertEquals(and, foldNull(and)); } @SuppressWarnings("unchecked") public void testNullFoldingDoesNotApplyOnAggregate() throws Exception { - FoldNull rule = new FoldNull(); - List> items = List.of(Max.class, Min.class); for (Class clazz : items) { Constructor ctor = clazz.getConstructor(Source.class, Expression.class); AggregateFunction conditionalFunction = ctor.newInstance(EMPTY, getFieldAttribute("a")); - assertEquals(conditionalFunction, rule.rule(conditionalFunction)); + assertEquals(conditionalFunction, foldNull(conditionalFunction)); conditionalFunction = ctor.newInstance(EMPTY, NULL); - assertEquals(NULL, rule.rule(conditionalFunction)); + assertEquals(NULL, foldNull(conditionalFunction)); } Avg avg = new Avg(EMPTY, getFieldAttribute("a")); - assertEquals(avg, rule.rule(avg)); + assertEquals(avg, foldNull(avg)); avg = new Avg(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(avg)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(avg)); Count count = new Count(EMPTY, getFieldAttribute("a")); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); count = new Count(EMPTY, NULL); - assertEquals(count, rule.rule(count)); + assertEquals(count, foldNull(count)); CountDistinct countd = new CountDistinct(EMPTY, getFieldAttribute("a"), getFieldAttribute("a")); - assertEquals(countd, rule.rule(countd)); + assertEquals(countd, foldNull(countd)); countd = new CountDistinct(EMPTY, NULL, NULL); - assertEquals(new Literal(EMPTY, null, LONG), rule.rule(countd)); + assertEquals(new Literal(EMPTY, null, LONG), foldNull(countd)); Median median = new Median(EMPTY, getFieldAttribute("a")); - assertEquals(median, rule.rule(median)); + assertEquals(median, foldNull(median)); median = new Median(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(median)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(median)); MedianAbsoluteDeviation medianad = new MedianAbsoluteDeviation(EMPTY, getFieldAttribute("a")); - assertEquals(medianad, rule.rule(medianad)); + assertEquals(medianad, foldNull(medianad)); medianad = new MedianAbsoluteDeviation(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(medianad)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(medianad)); Percentile percentile = new Percentile(EMPTY, getFieldAttribute("a"), getFieldAttribute("a")); - assertEquals(percentile, rule.rule(percentile)); + assertEquals(percentile, foldNull(percentile)); percentile = new Percentile(EMPTY, NULL, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(percentile)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(percentile)); Sum sum = new Sum(EMPTY, getFieldAttribute("a")); - assertEquals(sum, rule.rule(sum)); + assertEquals(sum, foldNull(sum)); sum = new Sum(EMPTY, NULL); - assertEquals(new Literal(EMPTY, null, DOUBLE), rule.rule(sum)); + assertEquals(new Literal(EMPTY, null, DOUBLE), foldNull(sum)); } public void testNullFoldableDoesNotApplyToIsNullAndNotNull() { - FoldNull rule = new FoldNull(); - DataType numericType = randomFrom(INTEGER, LONG, DOUBLE); DataType genericType = randomFrom(INTEGER, LONG, DOUBLE, UNSIGNED_LONG, KEYWORD, TEXT, GEO_POINT, GEO_SHAPE, VERSION, IP); List items = List.of( @@ -260,29 +255,26 @@ public void testNullFoldableDoesNotApplyToIsNullAndNotNull() { ); for (Expression item : items) { Expression isNull = new IsNull(EMPTY, item); - Expression transformed = rule.rule(isNull); + Expression transformed = foldNull(isNull); assertEquals(isNull, transformed); IsNotNull isNotNull = new IsNotNull(EMPTY, item); - transformed = rule.rule(isNotNull); + transformed = foldNull(isNotNull); assertEquals(isNotNull, transformed); } } public void testNullBucketGetsFolded() { - FoldNull foldNull = new FoldNull(); - assertEquals(NULL, foldNull.rule(new Bucket(EMPTY, NULL, NULL, NULL, NULL))); + assertEquals(NULL, foldNull(new Bucket(EMPTY, NULL, NULL, NULL, NULL))); } public void testNullCategorizeGroupingNotFolded() { - FoldNull foldNull = new FoldNull(); Categorize categorize = new Categorize(EMPTY, NULL); - assertEquals(categorize, foldNull.rule(categorize)); + assertEquals(categorize, foldNull(categorize)); } private void assertNullLiteral(Expression expression) { - assertEquals(Literal.class, expression.getClass()); - assertNull(expression.fold()); + assertNull(as(expression, Literal.class).value()); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java index 17e69e81444c5..1664e9f4653bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/LiteralsOnTheRightTests.java @@ -15,6 +15,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.FIVE; import static org.elasticsearch.xpack.esql.EsqlTestUtils.equalsOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; @@ -22,7 +23,7 @@ public class LiteralsOnTheRightTests extends ESTestCase { public void testLiteralsOnTheRight() { Alias a = new Alias(EMPTY, "a", new Literal(EMPTY, 10, INTEGER)); - Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a)); + Expression result = new LiteralsOnTheRight().rule(equalsOf(FIVE, a), unboundLogicalOptimizerContext()); assertTrue(result instanceof Equals); Equals eq = (Equals) result; assertEquals(a, eq.left()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java index 55091653e75d4..a6c0d838b2c21 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateEqualsTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; import org.elasticsearch.xpack.esql.core.expression.predicate.Range; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -37,11 +38,15 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOrEqualOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.notEqualsOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.rangeOf; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.expression.Literal.TRUE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class PropagateEqualsTests extends ESTestCase { + private Expression propagateEquals(BinaryLogic e) { + return new PropagateEquals().rule(e, unboundLogicalOptimizerContext()); + } // a == 1 AND a == 2 -> FALSE public void testDualEqualsConjunction() { @@ -49,8 +54,7 @@ public void testDualEqualsConjunction() { Equals eq1 = equalsOf(fa, ONE); Equals eq2 = equalsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, eq2)); + Expression exp = propagateEquals(new And(EMPTY, eq1, eq2)); assertEquals(FALSE, exp); } @@ -60,8 +64,7 @@ public void testEliminateRangeByEqualsOutsideInterval() { Equals eq1 = equalsOf(fa, new Literal(EMPTY, 10, DataType.INTEGER)); Range r = rangeOf(fa, ONE, false, new Literal(EMPTY, 10, DataType.INTEGER), false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, r)); + Expression exp = propagateEquals(new And(EMPTY, eq1, r)); assertEquals(FALSE, exp); } @@ -71,8 +74,7 @@ public void testPropagateEquals_VarNeq3AndVarEq3() { NotEquals neq = notEqualsOf(fa, THREE); Equals eq = equalsOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, neq, eq)); + Expression exp = propagateEquals(new And(EMPTY, neq, eq)); assertEquals(FALSE, exp); } @@ -82,8 +84,7 @@ public void testPropagateEquals_VarNeq4AndVarEq3() { NotEquals neq = notEqualsOf(fa, FOUR); Equals eq = equalsOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, neq, eq)); + Expression exp = propagateEquals(new And(EMPTY, neq, eq)); assertEquals(Equals.class, exp.getClass()); assertEquals(eq, exp); } @@ -94,8 +95,7 @@ public void testPropagateEquals_VarEq2AndVarLt2() { Equals eq = equalsOf(fa, TWO); LessThan lt = lessThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(FALSE, exp); } @@ -105,8 +105,7 @@ public void testPropagateEquals_VarEq2AndVarLte2() { Equals eq = equalsOf(fa, TWO); LessThanOrEqual lt = lessThanOrEqualOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(eq, exp); } @@ -116,8 +115,7 @@ public void testPropagateEquals_VarEq2AndVarLte1() { Equals eq = equalsOf(fa, TWO); LessThanOrEqual lt = lessThanOrEqualOf(fa, ONE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, lt)); + Expression exp = propagateEquals(new And(EMPTY, eq, lt)); assertEquals(FALSE, exp); } @@ -127,8 +125,7 @@ public void testPropagateEquals_VarEq2AndVarGt2() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gt)); + Expression exp = propagateEquals(new And(EMPTY, eq, gt)); assertEquals(FALSE, exp); } @@ -138,8 +135,7 @@ public void testPropagateEquals_VarEq2AndVarGte2() { Equals eq = equalsOf(fa, TWO); GreaterThanOrEqual gte = greaterThanOrEqualOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gte)); + Expression exp = propagateEquals(new And(EMPTY, eq, gte)); assertEquals(eq, exp); } @@ -149,8 +145,7 @@ public void testPropagateEquals_VarEq2AndVarLt3() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq, gt)); + Expression exp = propagateEquals(new And(EMPTY, eq, gt)); assertEquals(FALSE, exp); } @@ -162,9 +157,8 @@ public void testPropagateEquals_VarEq2AndVarLt3AndVarGt1AndVarNeq4() { GreaterThan gt = greaterThanOf(fa, ONE); NotEquals neq = notEqualsOf(fa, FOUR); - PropagateEquals rule = new PropagateEquals(); Expression and = Predicates.combineAnd(asList(eq, lt, gt, neq)); - Expression exp = rule.rule((And) and); + Expression exp = propagateEquals((And) and); assertEquals(eq, exp); } @@ -176,9 +170,8 @@ public void testPropagateEquals_VarEq2AndVarRangeGt1Lt3AndVarGt0AndVarNeq4() { GreaterThan gt = greaterThanOf(fa, new Literal(EMPTY, 0, DataType.INTEGER)); NotEquals neq = notEqualsOf(fa, FOUR); - PropagateEquals rule = new PropagateEquals(); Expression and = Predicates.combineAnd(asList(eq, range, gt, neq)); - Expression exp = rule.rule((And) and); + Expression exp = propagateEquals((And) and); assertEquals(eq, exp); } @@ -188,8 +181,7 @@ public void testPropagateEquals_VarEq2OrVarGt1() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, ONE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, gt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, gt)); assertEquals(gt, exp); } @@ -199,8 +191,7 @@ public void testPropagateEquals_VarEq2OrVarGte2() { Equals eq = equalsOf(fa, TWO); GreaterThan gt = greaterThanOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, gt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, gt)); assertEquals(GreaterThanOrEqual.class, exp.getClass()); GreaterThanOrEqual gte = (GreaterThanOrEqual) exp; assertEquals(TWO, gte.right()); @@ -212,8 +203,7 @@ public void testPropagateEquals_VarEq2OrVarLt3() { Equals eq = equalsOf(fa, TWO); LessThan lt = lessThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, lt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, lt)); assertEquals(lt, exp); } @@ -223,8 +213,7 @@ public void testPropagateEquals_VarEq3OrVarLt3() { Equals eq = equalsOf(fa, THREE); LessThan lt = lessThanOf(fa, THREE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, lt)); + Expression exp = propagateEquals(new Or(EMPTY, eq, lt)); assertEquals(LessThanOrEqual.class, exp.getClass()); LessThanOrEqual lte = (LessThanOrEqual) exp; assertEquals(THREE, lte.right()); @@ -236,8 +225,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt1Lt3() { Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, ONE, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(range, exp); } @@ -247,8 +235,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt2Lt3() { Equals eq = equalsOf(fa, TWO); Range range = rangeOf(fa, TWO, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(Range.class, exp.getClass()); Range r = (Range) exp; assertEquals(TWO, r.lower()); @@ -263,8 +250,7 @@ public void testPropagateEquals_VarEq3OrVarRangeGt2Lt3() { Equals eq = equalsOf(fa, THREE); Range range = rangeOf(fa, TWO, false, THREE, false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, range)); + Expression exp = propagateEquals(new Or(EMPTY, eq, range)); assertEquals(Range.class, exp.getClass()); Range r = (Range) exp; assertEquals(TWO, r.lower()); @@ -279,8 +265,7 @@ public void testPropagateEquals_VarEq2OrVarNeq2() { Equals eq = equalsOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, neq)); + Expression exp = propagateEquals(new Or(EMPTY, eq, neq)); assertEquals(TRUE, exp); } @@ -290,8 +275,7 @@ public void testPropagateEquals_VarEq2OrVarNeq5() { Equals eq = equalsOf(fa, TWO); NotEquals neq = notEqualsOf(fa, FIVE); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new Or(EMPTY, eq, neq)); + Expression exp = propagateEquals(new Or(EMPTY, eq, neq)); assertEquals(NotEquals.class, exp.getClass()); NotEquals ne = (NotEquals) exp; assertEquals(FIVE, ne.right()); @@ -305,8 +289,7 @@ public void testPropagateEquals_VarEq2OrVarRangeGt3Lt4OrVarGt2OrVarNe2() { GreaterThan gt = greaterThanOf(fa, TWO); NotEquals neq = notEqualsOf(fa, TWO); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule((Or) Predicates.combineOr(asList(eq, range, neq, gt))); + Expression exp = propagateEquals((Or) Predicates.combineOr(asList(eq, range, neq, gt))); assertEquals(TRUE, exp); } @@ -317,8 +300,7 @@ public void testPropagateEquals_ignoreDateTimeFields() { Equals eq2 = equalsOf(fa, TWO); And and = new And(EMPTY, eq1, eq2); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(and); + Expression exp = propagateEquals(and); assertEquals(and, exp); } @@ -328,8 +310,7 @@ public void testEliminateRangeByEqualsInInterval() { Equals eq1 = equalsOf(fa, ONE); Range r = rangeOf(fa, ONE, true, new Literal(EMPTY, 10, DataType.INTEGER), false); - PropagateEquals rule = new PropagateEquals(); - Expression exp = rule.rule(new And(EMPTY, eq1, r)); + Expression exp = propagateEquals(new And(EMPTY, eq1, r)); assertEquals(eq1, exp); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java index d1d6a7fbaa208..d35890e5b56bb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateNullableTests.java @@ -31,11 +31,20 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.greaterThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.lessThanOf; import static org.elasticsearch.xpack.esql.EsqlTestUtils.relation; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.expression.Literal.FALSE; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; public class PropagateNullableTests extends ESTestCase { + private Expression propagateNullable(And e) { + return new PropagateNullable().rule(e, unboundLogicalOptimizerContext()); + } + + private LogicalPlan propagateNullable(LogicalPlan p) { + return new PropagateNullable().apply(p, unboundLogicalOptimizerContext()); + } + private Literal nullOf(DataType dataType) { return new Literal(Source.EMPTY, null, dataType); } @@ -45,7 +54,7 @@ public void testIsNullAndNotNull() { FieldAttribute fa = getFieldAttribute(); And and = new And(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); - assertEquals(FALSE, new PropagateNullable().rule(and)); + assertEquals(FALSE, propagateNullable(and)); } // a IS NULL AND b IS NOT NULL AND c IS NULL AND d IS NOT NULL AND e IS NULL AND a IS NOT NULL => false @@ -58,7 +67,7 @@ public void testIsNullAndNotNullMultiField() { And and = new And(EMPTY, andOne, new And(EMPTY, andThree, andTwo)); - assertEquals(FALSE, new PropagateNullable().rule(and)); + assertEquals(FALSE, propagateNullable(and)); } // a IS NULL AND a > 1 => a IS NULL AND false @@ -67,7 +76,7 @@ public void testIsNullAndComparison() { IsNull isNull = new IsNull(EMPTY, fa); And and = new And(EMPTY, isNull, greaterThanOf(fa, ONE)); - assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), new PropagateNullable().rule(and)); + assertEquals(new And(EMPTY, isNull, nullOf(BOOLEAN)), propagateNullable(and)); } // a IS NULL AND b < 1 AND c < 1 AND a < 1 => a IS NULL AND b < 1 AND c < 1 => a IS NULL AND b < 1 AND c < 1 @@ -79,7 +88,7 @@ public void testIsNullAndMultipleComparison() { And and = new And(EMPTY, isNull, nestedAnd); And top = new And(EMPTY, and, lessThanOf(fa, ONE)); - Expression optimized = new PropagateNullable().rule(top); + Expression optimized = propagateNullable(top); Expression expected = new And(EMPTY, and, nullOf(BOOLEAN)); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); } @@ -97,7 +106,7 @@ public void testIsNullAndDeeplyNestedExpression() { Expression kept = new And(EMPTY, isNull, lessThanOf(getFieldAttribute("b"), THREE)); And and = new And(EMPTY, nullified, kept); - Expression optimized = new PropagateNullable().rule(and); + Expression optimized = propagateNullable(and); Expression expected = new And(EMPTY, new And(EMPTY, nullOf(BOOLEAN), nullOf(BOOLEAN)), kept); assertEquals(Predicates.splitAnd(expected), Predicates.splitAnd(optimized)); @@ -110,13 +119,13 @@ public void testIsNullInDisjunction() { Or or = new Or(EMPTY, new IsNull(EMPTY, fa), new IsNotNull(EMPTY, fa)); Filter dummy = new Filter(EMPTY, relation(), or); - LogicalPlan transformed = new PropagateNullable().apply(dummy); + LogicalPlan transformed = propagateNullable(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); or = new Or(EMPTY, new IsNull(EMPTY, fa), greaterThanOf(fa, ONE)); dummy = new Filter(EMPTY, relation(), or); - transformed = new PropagateNullable().apply(dummy); + transformed = propagateNullable(dummy); assertSame(dummy, transformed); assertEquals(or, ((Filter) transformed).condition()); } @@ -129,7 +138,7 @@ public void testIsNullDisjunction() { Or or = new Or(EMPTY, isNull, greaterThanOf(fa, THREE)); And and = new And(EMPTY, new Add(EMPTY, fa, ONE), or); - assertEquals(and, new PropagateNullable().rule(and)); + assertEquals(and, propagateNullable(and)); } public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() { @@ -141,7 +150,7 @@ public void testDoNotOptimizeIsNullAndMultipleComparisonWithConstants() { And aIsNull_AND_bLT1_AND_cLT1 = new And(EMPTY, aIsNull, bLT1_AND_cLT1); And aIsNull_AND_bLT1_AND_cLT1_AND_aLT1 = new And(EMPTY, aIsNull_AND_bLT1_AND_cLT1, lessThanOf(a, ONE)); - Expression optimized = new PropagateNullable().rule(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1); + Expression optimized = propagateNullable(aIsNull_AND_bLT1_AND_cLT1_AND_aLT1); Literal nullLiteral = new Literal(EMPTY, null, BOOLEAN); assertEquals(asList(aIsNull, nullLiteral, nullLiteral, nullLiteral), Predicates.splitAnd(optimized)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java index c7206c6971bde..b9ffc39e5e130 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/ReplaceRegexMatchTests.java @@ -10,8 +10,10 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; @@ -20,16 +22,20 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.EsqlTestUtils.getFieldAttribute; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; public class ReplaceRegexMatchTests extends ESTestCase { + private Expression replaceRegexMatch(RegexMatch e) { + return new ReplaceRegexMatch().rule(e, unboundLogicalOptimizerContext()); + } public void testMatchAllWildcardLikeToExist() { for (String s : asList("*", "**", "***")) { WildcardPattern pattern = new WildcardPattern(s); FieldAttribute fa = getFieldAttribute(); WildcardLike l = new WildcardLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(IsNotNull.class, e.getClass()); IsNotNull inn = (IsNotNull) e; assertEquals(fa, inn.field()); @@ -40,7 +46,7 @@ public void testMatchAllRLikeToExist() { RLikePattern pattern = new RLikePattern(".*"); FieldAttribute fa = getFieldAttribute(); RLike l = new RLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(IsNotNull.class, e.getClass()); IsNotNull inn = (IsNotNull) e; assertEquals(fa, inn.field()); @@ -51,11 +57,11 @@ public void testExactMatchWildcardLike() { WildcardPattern pattern = new WildcardPattern(s); FieldAttribute fa = getFieldAttribute(); WildcardLike l = new WildcardLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); - assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold()); + assertEquals(s.replace("\\", StringUtils.EMPTY), eq.right().fold(FoldContext.small())); } } @@ -63,11 +69,11 @@ public void testExactMatchRLike() { RLikePattern pattern = new RLikePattern("abc"); FieldAttribute fa = getFieldAttribute(); RLike l = new RLike(EMPTY, fa, pattern); - Expression e = new ReplaceRegexMatch().rule(l); + Expression e = replaceRegexMatch(l); assertEquals(Equals.class, e.getClass()); Equals eq = (Equals) e; assertEquals(fa, eq.left()); - assertEquals("abc", eq.right().fold()); + assertEquals("abc", eq.right().fold(FoldContext.small())); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java index 2429bcb1a1b04..90c8ae1032325 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/PushTopNToSourceTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.Nullability; @@ -417,7 +418,7 @@ private static void assertNoPushdownSort(TestPhysicalPlanBuilder builder, String private static PhysicalPlan pushTopNToSource(TopNExec topNExec) { var configuration = EsqlTestUtils.configuration("from test"); - var ctx = new LocalPhysicalOptimizerContext(configuration, SearchStats.EMPTY); + var ctx = new LocalPhysicalOptimizerContext(configuration, FoldContext.small(), SearchStats.EMPTY); var pushTopNToSource = new PushTopNToSource(); return pushTopNToSource.rule(topNExec, ctx); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java index e6fef186721a0..99a04b6ed8f10 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -151,8 +151,12 @@ void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, expectInvalidIndexNameErrorWithLineNumber(query, "\"" + indexString + "\"", lineNumber, indexString); } - void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String error) { - expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + error); + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name) { + expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name); + } + + void expectInvalidIndexNameErrorWithLineNumber(String query, String indexString, String lineNumber, String name, String error) { + expectError(LoggerMessageFormat.format(null, query, indexString), lineNumber + "Invalid index name [" + name + "], " + error); } void expectDateMathErrorWithLineNumber(String query, String arg, String lineNumber, String error) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java index 710637c05a900..85d4017b166fa 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/ExpressionTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; @@ -617,16 +618,14 @@ public void testSimplifyInWithSingleElementList() { Equals eq = (Equals) e; assertThat(eq.left(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.left()).name(), equalTo("a")); - assertThat(eq.right(), instanceOf(Literal.class)); - assertThat(eq.right().fold(), equalTo(1)); + assertThat(as(eq.right(), Literal.class).value(), equalTo(1)); e = whereExpression("1 IN (a)"); assertThat(e, instanceOf(Equals.class)); eq = (Equals) e; assertThat(eq.right(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a")); - assertThat(eq.left(), instanceOf(Literal.class)); - assertThat(eq.left().fold(), equalTo(1)); + assertThat(eq.left().fold(FoldContext.small()), equalTo(1)); e = whereExpression("1 NOT IN (a)"); assertThat(e, instanceOf(Not.class)); @@ -635,9 +634,7 @@ public void testSimplifyInWithSingleElementList() { eq = (Equals) e; assertThat(eq.right(), instanceOf(UnresolvedAttribute.class)); assertThat(((UnresolvedAttribute) eq.right()).name(), equalTo("a")); - assertThat(eq.left(), instanceOf(Literal.class)); - assertThat(eq.left().fold(), equalTo(1)); - + assertThat(eq.left().fold(FoldContext.small()), equalTo(1)); } private Expression whereExpression(String e) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index b83892ea47049..a6243d25ba579 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; @@ -577,6 +578,9 @@ public void testInvalidCharacterInIndexPattern() { expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, --indexpattern", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "indexpattern, \"--indexpattern\"", lineNumber, "-indexpattern"); expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern, --indexpattern\"", commands.get(command), "-indexpattern"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"- , -\"", commands.get(command), "", "must not be empty"); + expectInvalidIndexNameErrorWithLineNumber(command, "\"indexpattern,-\"", commands.get(command), "", "must not be empty"); + clustersAndIndices(command, "indexpattern", "*-"); clustersAndIndices(command, "indexpattern", "-indexpattern"); } @@ -621,11 +625,8 @@ public void testInvalidQuotingAsFromIndexPattern() { expectError("FROM \"foo\"bar\"", ": token recognition error at: '\"'"); expectError("FROM \"foo\"\"bar\"", ": extraneous input '\"bar\"' expecting "); - expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}"); - expectError( - "FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", - ": mismatched input '\"bar\"' expecting {, '|', ',', OPENING_BRACKET, 'metadata'}" - ); + expectError("FROM \"\"\"foo\"\"\"bar\"\"\"", ": mismatched input 'bar' expecting {, '|', ',', 'metadata'}"); + expectError("FROM \"\"\"foo\"\"\"\"\"\"bar\"\"\"", ": mismatched input '\"bar\"' expecting {, '|', ',', 'metadata'}"); } public void testInvalidQuotingAsMetricsIndexPattern() { @@ -905,10 +906,7 @@ public void testDeprecatedIsNullFunction() { public void testMetadataFieldOnOtherSources() { expectError("row a = 1 metadata _index", "line 1:20: extraneous input '_index' expecting "); expectError("show info metadata _index", "line 1:11: token recognition error at: 'm'"); - expectError( - "explain [from foo] metadata _index", - "line 1:20: mismatched input 'metadata' expecting {'|', ',', OPENING_BRACKET, ']', 'metadata'}" - ); + expectError("explain [from foo] metadata _index", "line 1:20: mismatched input 'metadata' expecting {'|', ',', ']', 'metadata'}"); } public void testMetadataFieldMultipleDeclarations() { @@ -1129,51 +1127,51 @@ public void testInputParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2")); + assertThat(alias.child().fold(FoldContext.small()), is("2")); field = row.fields().get(2); assertThat(field.name(), is("a")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("2 days")); + assertThat(alias.child().fold(FoldContext.small()), is("2 days")); field = row.fields().get(3); assertThat(field.name(), is("b")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is("4 hours")); + assertThat(alias.child().fold(FoldContext.small()), is("4 hours")); field = row.fields().get(4); assertThat(field.name(), is("c")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("1.2.3")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("1.2.3")); field = row.fields().get(5); assertThat(field.name(), is("d")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold().getClass(), is(String.class)); - assertThat(alias.child().fold().toString(), is("127.0.0.1")); + assertThat(alias.child().fold(FoldContext.small()).getClass(), is(String.class)); + assertThat(alias.child().fold(FoldContext.small()).toString(), is("127.0.0.1")); field = row.fields().get(6); assertThat(field.name(), is("e")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(9)); + assertThat(alias.child().fold(FoldContext.small()), is(9)); field = row.fields().get(7); assertThat(field.name(), is("f")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(11)); + assertThat(alias.child().fold(FoldContext.small()), is(11)); } public void testMissingInputParams() { @@ -1190,13 +1188,13 @@ public void testNamedParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidNamedParams() { @@ -1237,13 +1235,13 @@ public void testPositionalParams() { assertThat(field.name(), is("x")); assertThat(field, instanceOf(Alias.class)); Alias alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); field = row.fields().get(1); assertThat(field.name(), is("y")); assertThat(field, instanceOf(Alias.class)); alias = (Alias) field; - assertThat(alias.child().fold(), is(1)); + assertThat(alias.child().fold(FoldContext.small()), is(1)); } public void testInvalidPositionalParams() { @@ -2057,7 +2055,7 @@ private void assertStringAsLookupIndexPattern(String string, String statement) { var plan = statement(statement); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo(string)); + assertThat(tableName.fold(FoldContext.small()), equalTo(string)); } public void testIdPatternUnquoted() throws Exception { @@ -2125,7 +2123,7 @@ public void testLookup() { var plan = statement(query); var lookup = as(plan, Lookup.class); var tableName = as(lookup.tableName(), Literal.class); - assertThat(tableName.fold(), equalTo("t")); + assertThat(tableName.fold(FoldContext.small()), equalTo("t")); assertThat(lookup.matchFields(), hasSize(1)); var matchField = as(lookup.matchFields().get(0), UnresolvedAttribute.class); assertThat(matchField.name(), equalTo("j")); @@ -2306,7 +2304,7 @@ public void testMatchOperatorConstantQueryString() { var match = (Match) filter.condition(); var matchField = (UnresolvedAttribute) match.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(match.query().fold(), equalTo("value")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); } public void testInvalidMatchOperator() { @@ -2341,7 +2339,7 @@ public void testMatchFunctionFieldCasting() { var toInteger = (ToInteger) function.children().get(0); var matchField = (UnresolvedAttribute) toInteger.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(function.children().get(1).fold(), equalTo("value")); + assertThat(function.children().get(1).fold(FoldContext.small()), equalTo("value")); } public void testMatchOperatorFieldCasting() { @@ -2351,6 +2349,13 @@ public void testMatchOperatorFieldCasting() { var toInteger = (ToInteger) match.field(); var matchField = (UnresolvedAttribute) toInteger.field(); assertThat(matchField.name(), equalTo("field")); - assertThat(match.query().fold(), equalTo("value")); + assertThat(match.query().fold(FoldContext.small()), equalTo("value")); + } + + public void testFailingMetadataWithSquareBrackets() { + expectError( + "FROM test [METADATA _index] | STATS count(*)", + "line 1:11: mismatched input '[' expecting {, '|', ',', 'metadata'}" + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java index a254207865ad5..2f47a672a68d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QueryPlanTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -42,7 +43,7 @@ public void testTransformWithExpressionTopLevel() throws Exception { assertEquals(Limit.class, transformed.getClass()); Limit l = (Limit) transformed; - assertEquals(24, l.limit().fold()); + assertEquals(24, l.limit().fold(FoldContext.small())); } public void testTransformWithExpressionTree() throws Exception { @@ -53,7 +54,7 @@ public void testTransformWithExpressionTree() throws Exception { assertEquals(OrderBy.class, transformed.getClass()); OrderBy order = (OrderBy) transformed; assertEquals(Limit.class, order.child().getClass()); - assertEquals(24, ((Limit) order.child()).limit().fold()); + assertEquals(24, ((Limit) order.child()).limit().fold(FoldContext.small())); } public void testTransformWithExpressionTopLevelInCollection() throws Exception { @@ -83,12 +84,12 @@ public void testForEachWithExpressionTopLevel() throws Exception { List list = new ArrayList<>(); project.forEachExpression(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(one.child().fold()), list); + assertEquals(singletonList(one.child().fold(FoldContext.small())), list); } public void testForEachWithExpressionTree() throws Exception { @@ -97,12 +98,12 @@ public void testForEachWithExpressionTree() throws Exception { List list = new ArrayList<>(); o.forEachExpressionDown(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(limit.limit().fold()), list); + assertEquals(singletonList(limit.limit().fold(FoldContext.small())), list); } public void testForEachWithExpressionTopLevelInCollection() throws Exception { @@ -129,12 +130,12 @@ public void testForEachWithExpressionTreeInCollection() throws Exception { List list = new ArrayList<>(); project.forEachExpression(Literal.class, l -> { - if (l.fold().equals(42)) { - list.add(l.fold()); + if (l.value().equals(42)) { + list.add(l.value()); } }); - assertEquals(singletonList(one.child().fold()), list); + assertEquals(singletonList(one.child().fold(FoldContext.small())), list); } public void testPlanExpressions() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java index 5a7547d011c0f..e2eb05b0c14d3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/EvalMapperTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -145,7 +146,7 @@ public void testEvaluatorSuppliers() { lb.append(LONG); Layout layout = lb.build(); - var supplier = EvalMapper.toEvaluator(expression, layout); + var supplier = EvalMapper.toEvaluator(FoldContext.small(), expression, layout); EvalOperator.ExpressionEvaluator evaluator1 = supplier.get(driverContext()); EvalOperator.ExpressionEvaluator evaluator2 = supplier.get(driverContext()); assertNotNull(evaluator1); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java index 55f32d07fc2cb..4191f42f08237 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/FilterTests.java @@ -30,7 +30,6 @@ import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; -import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizer; @@ -52,6 +51,7 @@ import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.SerializationTestUtils.assertSerialization; import static org.elasticsearch.xpack.esql.core.util.Queries.Clause.FILTER; @@ -78,7 +78,7 @@ public static void init() { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(EsqlTestUtils.TEST_CFG)); + logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); physicalPlanOptimizer = new PhysicalPlanOptimizer(new PhysicalOptimizerContext(EsqlTestUtils.TEST_CFG)); mapper = new Mapper(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java index 5d8da21c6faad..a1648c67d9bd4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -83,6 +84,7 @@ public void closeIndex() throws IOException { public void testLuceneSourceOperatorHugeRowSize() throws IOException { int estimatedRowSize = randomEstimatedRowSize(estimatedRowSizeIsHuge); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, null, null, estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -98,6 +100,7 @@ public void testLuceneTopNSourceOperator() throws IOException { EsQueryExec.FieldSort sort = new EsQueryExec.FieldSort(sortField, Order.OrderDirection.ASC, Order.NullsPosition.LAST); Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -113,6 +116,7 @@ public void testLuceneTopNSourceOperatorDistanceSort() throws IOException { EsQueryExec.GeoDistanceSort sort = new EsQueryExec.GeoDistanceSort(sortField, Order.OrderDirection.ASC, 1, -1); Literal limit = new Literal(Source.EMPTY, 10, DataType.INTEGER); LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan( + FoldContext.small(), new EsQueryExec(Source.EMPTY, index(), IndexMode.STANDARD, List.of(), null, limit, List.of(sort), estimatedRowSize) ); assertThat(plan.driverFactories.size(), lessThanOrEqualTo(pragmas.taskConcurrency())); @@ -187,7 +191,7 @@ private EsPhysicalOperationProviders esPhysicalOperationProviders() throws IOExc ); } releasables.add(searcher); - return new EsPhysicalOperationProviders(shardContexts, null); + return new EsPhysicalOperationProviders(FoldContext.small(), shardContexts, null); } private IndexReader reader() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 01dd4db123ee2..628737aa36c6c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -41,6 +41,7 @@ import org.elasticsearch.xpack.esql.TestBlockFactory; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes; @@ -71,13 +72,13 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { private final List indexPages; - private TestPhysicalOperationProviders(List indexPages, AnalysisRegistry analysisRegistry) { - super(analysisRegistry); + private TestPhysicalOperationProviders(FoldContext foldContext, List indexPages, AnalysisRegistry analysisRegistry) { + super(foldContext, analysisRegistry); this.indexPages = indexPages; } - public static TestPhysicalOperationProviders create(List indexPages) throws IOException { - return new TestPhysicalOperationProviders(indexPages, createAnalysisRegistry()); + public static TestPhysicalOperationProviders create(FoldContext foldContext, List indexPages) throws IOException { + return new TestPhysicalOperationProviders(foldContext, indexPages, createAnalysisRegistry()); } public record IndexPage(String index, Page page, List columnNames) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java index f2a619f0dbd89..f3b1d84e507a5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/ClusterRequestTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; -import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; @@ -39,10 +38,10 @@ import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomConfiguration; import static org.elasticsearch.xpack.esql.ConfigurationTestUtils.randomTables; -import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.equalTo; @@ -187,7 +186,7 @@ static LogicalPlan parse(String query) { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(unboundLogicalOptimizerContext()); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()), TEST_VERIFIER diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java index 2cc733c2ea2e3..fac3495697da8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plugin/DataNodeRequestSerializationTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -289,7 +290,7 @@ static LogicalPlan parse(String query) { Map mapping = loadMapping("mapping-basic.json"); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); IndexResolution getIndexResult = IndexResolution.valid(test); - var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG)); + var logicalOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(TEST_CFG, FoldContext.small())); var analyzer = new Analyzer( new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, emptyPolicyResolution()), TEST_VERIFIER diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index 539cd0314a4d1..a3c5cd9168b4f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.action.EsqlResolveFieldsAction; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; +import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.session.EsqlSession; @@ -119,6 +120,7 @@ public void testFailedMetric() { request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, + FoldContext.small(), enrichResolver, new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, @@ -149,6 +151,7 @@ public void onFailure(Exception e) { request, randomAlphaOfLength(10), EsqlTestUtils.TEST_CFG, + FoldContext.small(), enrichResolver, new EsqlExecutionInfo(randomBoolean()), groupIndicesByCluster, diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java index 05b75fe6b01ca..0b106f6e304da 100644 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java +++ b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/FrozenIndices.java @@ -8,36 +8,22 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.frozen.FrozenEngine; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; import org.elasticsearch.xpack.frozen.action.TransportFreezeIndexAction; -import org.elasticsearch.xpack.frozen.rest.action.RestFreezeIndexAction; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.function.Predicate; -import java.util.function.Supplier; public class FrozenIndices extends Plugin implements ActionPlugin, EnginePlugin { @@ -63,19 +49,4 @@ public List> getSettings() { actions.add(new ActionHandler<>(FreezeIndexAction.INSTANCE, TransportFreezeIndexAction.class)); return actions; } - - @Override - public List getRestHandlers( - Settings settings, - NamedWriteableRegistry namedWriteableRegistry, - RestController restController, - ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, - SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster, - Predicate clusterSupportsFeature - ) { - return Collections.singletonList(new RestFreezeIndexAction()); - } } diff --git a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java b/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java deleted file mode 100644 index f07d17fee87f5..0000000000000 --- a/x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/xpack/frozen/rest/action/RestFreezeIndexAction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.frozen.rest.action; - -import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.internal.node.NodeClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.core.UpdateForV9; -import org.elasticsearch.protocol.xpack.frozen.FreezeRequest; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; - -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.POST; -import static org.elasticsearch.rest.RestUtils.getAckTimeout; -import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; - -public final class RestFreezeIndexAction extends BaseRestHandler { - - private static final String FREEZE_REMOVED = "It is no longer possible to freeze indices, but existing frozen indices can still be " - + "unfrozen"; - - private static final String UNFREEZE_DEPRECATED = "Frozen indices are deprecated because they provide no benefit given improvements " - + "in heap memory utilization. They will be removed in a future release."; - - @UpdateForV9(owner = UpdateForV9.Owner.DISTRIBUTED_INDEXING) - // these routes were ".deprecated" in RestApiVersion.V_8 which will require use of REST API compatibility headers to access - // this API in v9. It is unclear if this was intentional for v9, and the code has been updated to ".deprecateAndKeep" which will - // continue to emit deprecations warnings but will not require any special headers to access the API in v9. - // Please review and update the code and tests as needed. The original code remains commented out below for reference. - @Override - public List routes() { - return List.of( - // Route.builder(POST, "/{index}/_unfreeze").deprecated(UNFREEZE_DEPRECATED, RestApiVersion.V_8).build() - Route.builder(POST, "/{index}/_unfreeze").deprecateAndKeep(UNFREEZE_DEPRECATED).build() - ); - } - - @Override - protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) { - final var freezeRequest = new FreezeRequest( - getMasterNodeTimeout(request), - getAckTimeout(request), - Strings.splitStringByCommaToArray(request.param("index")) - ); - freezeRequest.indicesOptions(IndicesOptions.fromRequest(request, freezeRequest.indicesOptions())); - String waitForActiveShards = request.param("wait_for_active_shards"); - if (waitForActiveShards != null) { - freezeRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); - } - freezeRequest.setFreeze(false); - return channel -> client.execute(FreezeIndexAction.INSTANCE, freezeRequest, new RestToXContentListener<>(channel)); - } - - @Override - public String getName() { - return "freeze_index"; - } -} diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java index f25028824b56e..5d4b837dc4d1d 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ClusterStateWaitThresholdBreachTests.java @@ -108,7 +108,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { String[] firstAttemptShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -118,7 +118,7 @@ public void testWaitInShrunkShardsAllocatedExceedsThreshold() throws Exception { // let's check ILM for the managed index is waiting in the `shrunk-shards-allocated` step assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -160,7 +160,7 @@ public void onFailure(Exception e) { String[] secondCycleShrinkIndexName = new String[1]; assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -180,7 +180,9 @@ public void onFailure(Exception e) { // situation and allow for shrink to complete by reducing the number of shards for the shrunk index to 0 setReplicaCount(0, secondCycleShrinkIndexName[0]); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(secondCycleShrinkIndexName[0]); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + secondCycleShrinkIndexName[0] + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses() .get(secondCycleShrinkIndexName[0]); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java index 21924634ff6ab..d9cc4de344a39 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataStreamAndIndexLifecycleMixingTests.java @@ -149,7 +149,10 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -193,7 +196,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -270,7 +273,7 @@ public void testIndexTemplateSwapsILMForDataStreamLifecycle() throws Exception { String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -348,7 +351,10 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -401,7 +407,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex @@ -465,7 +471,7 @@ public void testUpdateIndexTemplateFromILMtoBothILMAndDataStreamLifecycle() thro String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -543,7 +549,10 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce List backingIndices = getBackingIndices(dataStreamName); String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(firstGenerationIndex, secondGenerationIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + firstGenerationIndex, + secondGenerationIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse firstGenerationExplain = explainResponse.getIndexResponses().get(firstGenerationIndex); @@ -591,7 +600,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String firstGenerationIndex = backingIndices.get(0); String secondGenerationIndex = backingIndices.get(1); String writeIndex = backingIndices.get(2); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, writeIndex @@ -663,7 +672,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -720,7 +729,7 @@ public void testUpdateIndexTemplateToDataStreamLifecyclePreference() throws Exce String secondGenerationIndex = backingIndices.get(1); String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices( + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( firstGenerationIndex, secondGenerationIndex, thirdGenerationIndex, @@ -846,7 +855,7 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw assertThat(index.isManagedByLifecycle(), is(true)); } - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(writeIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse writeIndexExplain = explainResponse.getIndexResponses().get(writeIndex); @@ -875,7 +884,10 @@ public void testUpdateIndexTemplateToMigrateFromDataStreamLifecycleToIlm() throw String thirdGenerationIndex = backingIndices.get(2); String writeIndex = backingIndices.get(3); - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(thirdGenerationIndex, writeIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices( + thirdGenerationIndex, + writeIndex + ); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse thirdGenerationExplain = explainResponse.getIndexResponses().get(thirdGenerationIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java index 6d409bf474cfc..d483401931830 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/DataTiersMigrationsTests.java @@ -116,7 +116,7 @@ public void testIndexDataTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -127,7 +127,7 @@ public void testIndexDataTierMigration() throws Exception { logger.info("starting a warm data node"); internalCluster().startNode(warmNode(Settings.EMPTY)); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -140,7 +140,7 @@ public void testIndexDataTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase after the index has been migrated to the cold node assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -177,7 +177,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { assertTrue(res.isAcknowledged()); assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -197,7 +197,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // 1. start another cold node so both the primary and replica can relocate to the cold nodes // 2. remove the tier routing setting from the index again (we're doing this below) assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); @@ -210,7 +210,7 @@ public void testUserOptsOutOfTierMigration() throws Exception { // wait for lifecycle to complete in the cold phase assertBusy(() -> { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest().indices(managedIndex); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices(managedIndex); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); IndexLifecycleExplainResponse indexLifecycleExplainResponse = explainResponse.getIndexResponses().get(managedIndex); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java index 2c4c1c9e20bb6..3f92abc7e3da2 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java index b91a309a23ae5..c695dc8f87b68 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/ILMMultiNodeWithCCRDisabledIT.java @@ -99,8 +99,10 @@ public void testShrinkOnTiers() throws Exception { prepareIndex(index).setCreate(true).setId("1").setSource("@timestamp", "2020-09-09").get(); assertBusy(() -> { - ExplainLifecycleResponse explain = client().execute(ExplainLifecycleAction.INSTANCE, new ExplainLifecycleRequest().indices("*")) - .get(); + ExplainLifecycleResponse explain = client().execute( + ExplainLifecycleAction.INSTANCE, + new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT).indices("*") + ).get(); logger.info("--> explain: {}", Strings.toString(explain)); String backingIndexName = DataStream.getDefaultBackingIndexName(index, 1); diff --git a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java index 644f88dc533b9..8999adc08f97b 100644 --- a/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java +++ b/x-pack/plugin/ilm/src/internalClusterTest/java/org/elasticsearch/xpack/ilm/IndexLifecycleInitialisationTests.java @@ -388,7 +388,7 @@ public void testExplainParseOriginationDate() throws Exception { private IndexLifecycleExplainResponse executeExplainRequestAndGetTestIndexResponse(String indexName) throws ExecutionException, InterruptedException { - ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainRequest = new ExplainLifecycleRequest(TEST_REQUEST_TIMEOUT); ExplainLifecycleResponse explainResponse = client().execute(ExplainLifecycleAction.INSTANCE, explainRequest).get(); assertThat(explainResponse.getIndexResponses().size(), equalTo(1)); return explainResponse.getIndexResponses().get(indexName); diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java index 195f989eab055..3950edf648e18 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/RestExplainLifecycleAction.java @@ -36,12 +36,11 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { String[] indexes = Strings.splitStringByCommaToArray(restRequest.param("index")); - ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(); + ExplainLifecycleRequest explainLifecycleRequest = new ExplainLifecycleRequest(getMasterNodeTimeout(restRequest)); explainLifecycleRequest.indices(indexes); explainLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(restRequest, IndicesOptions.strictExpandOpen())); explainLifecycleRequest.onlyManaged(restRequest.paramAsBoolean("only_managed", false)); explainLifecycleRequest.onlyErrors(restRequest.paramAsBoolean("only_errors", false)); - explainLifecycleRequest.masterNodeTimeout(getMasterNodeTimeout(restRequest)); return channel -> client.execute(ExplainLifecycleAction.INSTANCE, explainLifecycleRequest, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java new file mode 100644 index 0000000000000..4400ad8bbb538 --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceBasicLicenseIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.ClassRule; + +import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig; + +public class InferenceBasicLicenseIT extends InferenceLicenseBaseRestTest { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "basic") + .setting("xpack.security.enabled", "true") + .user("x_pack_rest_user", "x-pack-test-password") + .plugin("inference-service-test") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testPutModel_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var modelConfig = mockSparseServiceModelConfig(null, true); + sendRestrictedRequest("PUT", endpoint, modelConfig); + } + + public void testUpdateModel_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "task_settings": { + "num_threads": 2 + } + } + """; + sendRestrictedRequest("PUT", endpoint, requestBody); + } + + public void testPerformInference_RestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "input": ["washing", "machine"] + } + """; + sendRestrictedRequest("POST", endpoint, requestBody); + } + + public void testGetServices_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_services"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testGetModels_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_all"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + sendNonRestrictedRequest("DELETE", endpoint, null, 404, true); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java new file mode 100644 index 0000000000000..43183bae73252 --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceLicenseBaseRestTest.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; + +public class InferenceLicenseBaseRestTest extends ESRestTestCase { + protected void sendRestrictedRequest(String method, String endpoint, String body) throws IOException { + var request = new Request(method, endpoint); + request.setJsonEntity(body); + + var exception = assertThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(403, exception.getResponse().getStatusLine().getStatusCode()); + assertThat(exception.getMessage(), containsString("current license is non-compliant for [inference]")); + } + + protected void sendNonRestrictedRequest(String method, String endpoint, String body, int expectedStatusCode, boolean exceptionExpected) + throws IOException { + var request = new Request(method, endpoint); + request.setJsonEntity(body); + + int actualStatusCode; + if (exceptionExpected) { + var exception = assertThrows(ResponseException.class, () -> client().performRequest(request)); + actualStatusCode = exception.getResponse().getStatusLine().getStatusCode(); + } else { + var response = client().performRequest(request); + actualStatusCode = response.getStatusLine().getStatusCode(); + } + assertEquals(expectedStatusCode, actualStatusCode); + } +} diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java new file mode 100644 index 0000000000000..c7066a827fa7e --- /dev/null +++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceTrialLicenseIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.junit.ClassRule; + +import static org.elasticsearch.xpack.inference.InferenceBaseRestTest.mockSparseServiceModelConfig; + +public class InferenceTrialLicenseIT extends InferenceLicenseBaseRestTest { + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .setting("xpack.license.self_generated.type", "trial") + .setting("xpack.security.enabled", "true") + .user("x_pack_rest_user", "x-pack-test-password") + .plugin("inference-service-test") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue("x_pack_rest_user", new SecureString("x-pack-test-password".toCharArray())); + return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + public void testPutModel_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var modelConfig = mockSparseServiceModelConfig(null, true); + sendNonRestrictedRequest("PUT", endpoint, modelConfig, 200, false); + } + + public void testUpdateModel_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s/_update?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "task_settings": { + "num_threads": 2 + } + } + """; + sendNonRestrictedRequest("PUT", endpoint, requestBody, 404, true); + } + + public void testPerformInference_NonRestrictedWithTrialLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + var requestBody = """ + { + "input": ["washing", "machine"] + } + """; + sendNonRestrictedRequest("POST", endpoint, requestBody, 404, true); + } + + public void testGetServices_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_services"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testGetModels_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = "_inference/_all"; + sendNonRestrictedRequest("GET", endpoint, null, 200, false); + } + + public void testDeleteModel_NonRestrictedWithBasicLicense() throws Exception { + var endpoint = Strings.format("_inference/%s/%s?error_trace", TaskType.SPARSE_EMBEDDING, "endpoint-id"); + sendNonRestrictedRequest("DELETE", endpoint, null, 404, true); + } +} diff --git a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java index d38503a884092..880557c59f11c 100644 --- a/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java +++ b/x-pack/plugin/inference/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/xpack/application/InferenceUpgradeTestCase.java @@ -12,8 +12,11 @@ import org.elasticsearch.client.Request; import org.elasticsearch.common.Strings; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; import org.elasticsearch.test.http.MockWebServer; -import org.elasticsearch.upgrades.AbstractRollingUpgradeTestCase; +import org.elasticsearch.upgrades.ParameterizedRollingUpgradeTestCase; +import org.junit.ClassRule; import java.io.IOException; import java.util.LinkedList; @@ -22,7 +25,7 @@ import static org.elasticsearch.core.Strings.format; -public class InferenceUpgradeTestCase extends AbstractRollingUpgradeTestCase { +public class InferenceUpgradeTestCase extends ParameterizedRollingUpgradeTestCase { static final String MODELS_RENAMED_TO_ENDPOINTS = "8.15.0"; @@ -30,6 +33,20 @@ public InferenceUpgradeTestCase(@Name("upgradedNodes") int upgradedNodes) { super(upgradedNodes); } + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .version(getOldClusterTestVersion()) + .nodes(NODE_NUM) + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected ElasticsearchCluster getUpgradeCluster() { + return cluster; + } + protected static String getUrl(MockWebServer webServer) { return format("http://%s:%s", webServer.getHostName(), webServer.getPort()); } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java index 90a9dd3355b3c..3f7a51cc4e9f9 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterIT.java @@ -27,6 +27,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; @@ -73,7 +74,7 @@ public void setup() throws Exception { @Override protected Collection> nodePlugins() { - return Arrays.asList(Utils.TestInferencePlugin.class); + return Arrays.asList(Utils.TestInferencePlugin.class, LocalStateCompositeXPackPlugin.class); } @Override diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java index be6b3725b0f35..24585318b15b3 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/ModelRegistryIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.chunking.ChunkingSettingsTests; import org.elasticsearch.xpack.inference.registry.ModelRegistry; @@ -76,7 +77,7 @@ public void createComponents() { @Override protected Collection> getPlugins() { - return pluginList(ReindexPlugin.class, InferencePlugin.class); + return pluginList(ReindexPlugin.class, InferencePlugin.class, LocalStateCompositeXPackPlugin.class); } public void testStoreModel() throws Exception { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java index cf75c1bfe640e..580e5dd4bf8a1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java @@ -52,7 +52,8 @@ public Set getTestFeatures() { SEMANTIC_MATCH_QUERY_REWRITE_INTERCEPTION_SUPPORTED, SEMANTIC_SPARSE_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES, - SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED + SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED, + TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX ); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java index b16c53a428d73..d303ead4d9188 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferencePlugin.java @@ -29,6 +29,8 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.inference.InferenceServiceExtension; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.node.PluginComponentBinding; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.ExtensiblePlugin; @@ -150,6 +152,12 @@ public class InferencePlugin extends Plugin implements ActionPlugin, ExtensibleP Setting.Property.Dynamic ); + public static final LicensedFeature.Momentary INFERENCE_API_FEATURE = LicensedFeature.momentary( + "inference", + "api", + License.OperationMode.ENTERPRISE + ); + public static final String NAME = "inference"; public static final String UTILITY_THREAD_POOL_NAME = "inference_utility"; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java index 2a0e8e1775279..b6c7d26b36f9a 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceAction.java @@ -23,9 +23,12 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -38,6 +41,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.modelAttributes; import static org.elasticsearch.xpack.inference.telemetry.InferenceStats.responseAttributes; @@ -48,6 +52,7 @@ public abstract class BaseTransportInferenceAction requestReader ) { super(inferenceActionName, transportService, actionFilters, requestReader, EsExecutors.DIRECT_EXECUTOR_SERVICE); + this.licenseState = licenseState; this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; this.inferenceStats = inferenceStats; @@ -72,6 +79,11 @@ public BaseTransportInferenceAction( @Override protected void doExecute(Task task, Request request, ActionListener listener) { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var timer = InferenceTimer.start(); var getModelListener = ActionListener.wrap((UnparsedModel unparsedModel) -> { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java index 08e6d869a553d..24ef0d7d610d0 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportInferenceAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.inference.Model; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -28,6 +29,7 @@ public class TransportInferenceAction extends BaseTransportInferenceAction listener ) throws Exception { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + listener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var requestAsMap = requestToMap(request); var resolvedTaskType = ServiceUtils.resolveTaskType(request.getTaskType(), (String) requestAsMap.remove(TaskType.NAME)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java index f0906231d8f42..1478130f6a6c8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionInferenceAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; @@ -30,6 +31,7 @@ public class TransportUnifiedCompletionInferenceAction extends BaseTransportInfe public TransportUnifiedCompletionInferenceAction( TransportService transportService, ActionFilters actionFilters, + XPackLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -39,6 +41,7 @@ public TransportUnifiedCompletionInferenceAction( UnifiedCompletionAction.NAME, transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java index 3c47de1ad64d1..4a90f73722438 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/TransportUpdateInferenceModelAction.java @@ -34,12 +34,15 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xpack.core.XPackField; import org.elasticsearch.xpack.core.inference.action.UpdateInferenceModelAction; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.UpdateTrainedModelDeploymentAction; @@ -57,6 +60,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import static org.elasticsearch.xpack.inference.InferencePlugin.INFERENCE_API_FEATURE; import static org.elasticsearch.xpack.inference.services.ServiceUtils.resolveTaskType; import static org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalServiceSettings.NUM_ALLOCATIONS; @@ -66,6 +70,7 @@ public class TransportUpdateInferenceModelAction extends TransportMasterNodeActi private static final Logger logger = LogManager.getLogger(TransportUpdateInferenceModelAction.class); + private final XPackLicenseState licenseState; private final ModelRegistry modelRegistry; private final InferenceServiceRegistry serviceRegistry; private final Client client; @@ -77,6 +82,7 @@ public TransportUpdateInferenceModelAction( ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + XPackLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, Client client @@ -92,6 +98,7 @@ public TransportUpdateInferenceModelAction( UpdateInferenceModelAction.Response::new, EsExecutors.DIRECT_EXECUTOR_SERVICE ); + this.licenseState = licenseState; this.modelRegistry = modelRegistry; this.serviceRegistry = serviceRegistry; this.client = client; @@ -104,6 +111,11 @@ protected void masterOperation( ClusterState state, ActionListener masterListener ) { + if (INFERENCE_API_FEATURE.check(licenseState) == false) { + masterListener.onFailure(LicenseUtils.newComplianceException(XPackField.INFERENCE)); + return; + } + var bodyTaskType = request.getContentAsSettings().taskType(); var resolvedTaskType = resolveTaskType(request.getTaskType(), bodyTaskType != null ? bodyTaskType.toString() : null); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java index 1fc61d3331d20..0804cb332b15d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntity.java @@ -39,7 +39,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = covertToString(taskSettings.getInputType()); + String inputType = convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } @@ -49,7 +49,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { if (inputType == null) { return null; } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java index 8fae9408b860d..22676c11dc548 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchSparseRequestEntity.java @@ -34,7 +34,7 @@ public record AlibabaCloudSearchSparseRequestEntity(List input, AlibabaC public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(TEXTS_FIELD, input); - String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(taskSettings.getInputType()); + String inputType = AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(taskSettings.getInputType()); if (inputType != null) { builder.field(INPUT_TYPE_FIELD, inputType); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java index 63cc5c3cb7261..a972cbbac959d 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntity.java @@ -51,7 +51,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } if (taskSettings.getInputType() != null) { - builder.field(INPUT_TYPE_FIELD, covertToString(taskSettings.getInputType())); + builder.field(INPUT_TYPE_FIELD, convertToString(taskSettings.getInputType())); } if (embeddingType != null) { @@ -67,7 +67,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } // default for testing - static String covertToString(InputType inputType) { + static String convertToString(InputType inputType) { return switch (inputType) { case INGEST -> SEARCH_DOCUMENT; case SEARCH -> SEARCH_QUERY; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java index 7236b0141a86d..503000c31f7e7 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilder.java @@ -14,6 +14,7 @@ import org.elasticsearch.search.retriever.RetrieverBuilder; import org.elasticsearch.search.retriever.RetrieverParserContext; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -48,7 +49,12 @@ public class RandomRankRetrieverBuilder extends RetrieverBuilder { }); static { - PARSER.declareNamedObject(constructorArg(), (p, c, n) -> p.namedObject(RetrieverBuilder.class, n, c), RETRIEVER_FIELD); + PARSER.declareField( + constructorArg(), + RandomRankRetrieverBuilder::parseRetrieverBuilder, + RETRIEVER_FIELD, + ObjectParser.ValueType.OBJECT + ); PARSER.declareString(optionalConstructorArg(), FIELD_FIELD); PARSER.declareInt(optionalConstructorArg(), RANK_WINDOW_SIZE_FIELD); PARSER.declareInt(optionalConstructorArg(), SEED_FIELD); @@ -63,6 +69,22 @@ public static RandomRankRetrieverBuilder fromXContent(XContentParser parser, Ret return PARSER.apply(parser, context); } + private static RetrieverBuilder parseRetrieverBuilder(XContentParser parser, RetrieverParserContext context) throws IOException { + assert parser.currentToken() == XContentParser.Token.START_OBJECT; + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.END_OBJECT) { + throw new ParsingException(parser.getTokenLocation(), "empty [" + RETRIEVER_FIELD + "] object"); + } + assert parser.currentToken() == XContentParser.Token.FIELD_NAME; + final RetrieverBuilder builder = parser.namedObject(RetrieverBuilder.class, parser.currentName(), context); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.FIELD_NAME) { + throw new ParsingException(parser.getTokenLocation(), "unexpected field [" + parser.currentName() + "]"); + } + assert parser.currentToken() == XContentParser.Token.END_OBJECT; + return builder; + } + private final RetrieverBuilder retrieverBuilder; private final String field; private final int rankWindowSize; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java index 1b6fb55659a87..bd4c9c1a8be42 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java @@ -44,6 +44,10 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder "text_similarity_reranker_retriever_composition_supported", true ); + public static final NodeFeature TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX = new NodeFeature( + "text_similarity_reranker_alias_handling_fix", + true + ); public static final ParseField RETRIEVER_FIELD = new ParseField("retriever"); public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id"); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java index bfec2d5ac3484..a595134ecd548 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import org.hamcrest.Matchers; @@ -28,7 +29,7 @@ public class SemanticTextClusterMetadataTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(InferencePlugin.class); + return List.of(InferencePlugin.class, LocalStateCompositeXPackPlugin.class); } public void testCreateIndexWithSemanticTextField() { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java index 47f3a0e0b57aa..a723e5a9dffdf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/BaseTransportInferenceActionTestCase.java @@ -18,11 +18,13 @@ import org.elasticsearch.inference.ModelConfigurations; import org.elasticsearch.inference.TaskType; import org.elasticsearch.inference.UnparsedModel; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.BaseInferenceActionRequest; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.inference.InferencePlugin; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; import org.elasticsearch.xpack.inference.registry.ModelRegistry; import org.elasticsearch.xpack.inference.telemetry.InferenceStats; @@ -49,6 +51,7 @@ import static org.mockito.Mockito.when; public abstract class BaseTransportInferenceActionTestCase extends ESTestCase { + private MockLicenseState licenseState; private ModelRegistry modelRegistry; private StreamingTaskManager streamingTaskManager; private BaseTransportInferenceAction action; @@ -64,16 +67,28 @@ public void setUp() throws Exception { super.setUp(); TransportService transportService = mock(); ActionFilters actionFilters = mock(); + licenseState = mock(); modelRegistry = mock(); serviceRegistry = mock(); inferenceStats = new InferenceStats(mock(), mock()); streamingTaskManager = mock(); - action = createAction(transportService, actionFilters, modelRegistry, serviceRegistry, inferenceStats, streamingTaskManager); + action = createAction( + transportService, + actionFilters, + licenseState, + modelRegistry, + serviceRegistry, + inferenceStats, + streamingTaskManager + ); + + mockValidLicenseState(); } protected abstract BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -361,4 +376,8 @@ protected void mockModelAndServiceRegistry(InferenceService service) { when(serviceRegistry.getService(any())).thenReturn(Optional.of(service)); } + + protected void mockValidLicenseState(){ + when(licenseState.isAllowed(InferencePlugin.INFERENCE_API_FEATURE)).thenReturn(true); + } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java index e54175cb27009..a5efe04c22c04 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportInferenceActionTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.inference.InferenceServiceRegistry; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.action.task.StreamingTaskManager; @@ -23,6 +24,7 @@ public class TransportInferenceActionTests extends BaseTransportInferenceActionT protected BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -31,6 +33,7 @@ protected BaseTransportInferenceAction createAction( return new TransportInferenceAction( transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java index 4c943599ce523..3856a3d111b6e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/TransportUnifiedCompletionActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.inference.InferenceServiceRegistry; import org.elasticsearch.inference.TaskType; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.inference.action.UnifiedCompletionAction; @@ -36,6 +37,7 @@ public class TransportUnifiedCompletionActionTests extends BaseTransportInferenc protected BaseTransportInferenceAction createAction( TransportService transportService, ActionFilters actionFilters, + MockLicenseState licenseState, ModelRegistry modelRegistry, InferenceServiceRegistry serviceRegistry, InferenceStats inferenceStats, @@ -44,6 +46,7 @@ protected BaseTransportInferenceAction createAc return new TransportUnifiedCompletionInferenceAction( transportService, actionFilters, + licenseState, modelRegistry, serviceRegistry, inferenceStats, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java index 6aaab219c331d..39eeef3a95520 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/alibabacloudsearch/AlibabaCloudSearchEmbeddingsRequestEntityTests.java @@ -50,7 +50,7 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { var thrownException = expectThrows( AssertionError.class, - () -> AlibabaCloudSearchEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED) + () -> AlibabaCloudSearchEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java index 0690bf56893ca..8c8aeba4a0a07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/external/request/cohere/CohereEmbeddingsRequestEntityTests.java @@ -84,7 +84,10 @@ public void testXContent_WritesNoOptionalFields_WhenTheyAreNotDefined() throws I } public void testConvertToString_ThrowsAssertionFailure_WhenInputTypeIsUnspecified() { - var thrownException = expectThrows(AssertionError.class, () -> CohereEmbeddingsRequestEntity.covertToString(InputType.UNSPECIFIED)); + var thrownException = expectThrows( + AssertionError.class, + () -> CohereEmbeddingsRequestEntity.convertToString(InputType.UNSPECIFIED) + ); MatcherAssert.assertThat(thrownException.getMessage(), is("received invalid input type value [unspecified]")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java index 1f58c4165056d..0025b3a53a69f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextNonDynamicFieldMapperTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.mapper.NonDynamicFieldMapperTests; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.Utils; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; import org.junit.Before; @@ -26,7 +27,7 @@ public void setup() throws Exception { @Override protected Collection> getPlugins() { - return List.of(Utils.TestInferencePlugin.class); + return List.of(Utils.TestInferencePlugin.class, LocalStateCompositeXPackPlugin.class); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java index c0ef4e45f101f..b4cc699334a82 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/random/RandomRankRetrieverBuilderTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.usage.SearchUsage; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.json.JsonXContent; @@ -23,6 +24,7 @@ import java.util.List; import static org.elasticsearch.search.rank.RankBuilder.DEFAULT_RANK_WINDOW_SIZE; +import static org.hamcrest.Matchers.containsString; public class RandomRankRetrieverBuilderTests extends AbstractXContentTestCase { @@ -49,10 +51,7 @@ protected RandomRankRetrieverBuilder createTestInstance() { protected RandomRankRetrieverBuilder doParseInstance(XContentParser parser) throws IOException { return (RandomRankRetrieverBuilder) RetrieverBuilder.parseTopLevelRetrieverBuilder( parser, - new RetrieverParserContext( - new SearchUsage(), - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED || nf == RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED - ) + new RetrieverParserContext(new SearchUsage(), nf -> nf == RandomRankRetrieverBuilder.RANDOM_RERANKER_RETRIEVER_SUPPORTED) ); } @@ -99,4 +98,65 @@ public void testParserDefaults() throws IOException { } } + public void testParserEmptyRetriever() throws IOException { + String json = """ + { + "retriever": { + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("empty [retriever] object")); + } + } + + public void testParserWrongRetrieverName() throws IOException { + String json = """ + { + "retriever": { + "test2": { + "value": "my-test-retriever" + } + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("unknown field [test2]")); + } + } + + public void testExtraContent() throws IOException { + String json = """ + { + "retriever": { + "test": { + "value": "my-test-retriever" + }, + "field2": "my-field" + }, + "field": "my-field" + }"""; + + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + XContentParseException ex = expectThrows( + XContentParseException.class, + () -> RandomRankRetrieverBuilder.PARSER.parse(parser, null) + ); + assertThat(ex.getMessage(), containsString("[random_reranker] failed to parse field [retriever]")); + assertThat(ex.getCause().getMessage(), containsString("unexpected field [field2]")); + } + } + } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java index 6d6403b69ea11..69b1e19fa91de 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankMultiNodeTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.rank.RankBuilder; import org.elasticsearch.search.rank.rerank.AbstractRerankerIT; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.inference.InferencePlugin; import java.util.Collection; @@ -40,7 +41,7 @@ protected RankBuilder getThrowingRankBuilder(int rankWindowSize, String rankFeat @Override protected Collection> pluginsNeeded() { - return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class); + return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class, LocalStateCompositeXPackPlugin.class); } public void testQueryPhaseShardThrowingAllShardsFail() throws Exception { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java index 478f3b2f33c93..b0fa1797490eb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilderTests.java @@ -70,8 +70,7 @@ protected TextSimilarityRankRetrieverBuilder doParseInstance(XContentParser pars parser, new RetrieverParserContext( new SearchUsage(), - nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED - || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED + nf -> nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_RETRIEVER_SUPPORTED || nf == TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_COMPOSITION_SUPPORTED ) ); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java index a042fca44fdb5..a6a4ce2b2ffdf 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.rank.rerank.AbstractRerankerIT; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.inference.InferencePlugin; import org.junit.Before; @@ -108,7 +109,7 @@ protected InferenceAction.Request generateRequest(List docFeatures) { @Override protected Collection> getPlugins() { - return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class); + return List.of(InferencePlugin.class, TextSimilarityTestPlugin.class, LocalStateCompositeXPackPlugin.class); } @Before diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java index 3a5f56c0b4247..167a3f2688292 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/alibabacloudsearch/completion/AlibabaCloudSearchCompletionServiceSettingsTests.java @@ -8,9 +8,7 @@ package org.elasticsearch.xpack.inference.services.alibabacloudsearch.completion; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.inference.services.ServiceFields; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettings; import org.elasticsearch.xpack.inference.services.alibabacloudsearch.AlibabaCloudSearchServiceSettingsTests; import org.hamcrest.MatcherAssert; @@ -29,10 +27,6 @@ public static AlibabaCloudSearchCompletionServiceSettings createRandom() { } public void testFromMap() { - var url = "https://www.abc.com"; - var similarity = SimilarityMeasure.DOT_PRODUCT.toString(); - var dims = 1536; - var maxInputTokens = 512; var model = "model"; var host = "host"; var workspaceName = "default"; @@ -40,14 +34,6 @@ public void testFromMap() { var serviceSettings = AlibabaCloudSearchCompletionServiceSettings.fromMap( new HashMap<>( Map.of( - ServiceFields.URL, - url, - ServiceFields.SIMILARITY, - similarity, - ServiceFields.DIMENSIONS, - dims, - ServiceFields.MAX_INPUT_TOKENS, - maxInputTokens, AlibabaCloudSearchServiceSettings.HOST, host, AlibabaCloudSearchServiceSettings.SERVICE_ID, diff --git a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml index 2980d42d22c3e..a3c55dfddf611 100644 --- a/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml +++ b/x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml @@ -216,3 +216,82 @@ setup: - close_to: { hits.hits.0._explanation.value: { value: 0.4, error: 0.000001 } } - match: {hits.hits.0._explanation.description: "/text_similarity_reranker.match.using.inference.endpoint:.\\[my-rerank-model\\].on.document.field:.\\[text\\].*/" } - match: {hits.hits.0._explanation.details.0.description: "/weight.*science.*/" } + +--- +"text similarity reranker properly handles aliases": + - requires: + cluster_features: "text_similarity_reranker_alias_handling_fix" + reason: Test for alias handling fix + + # Create an empty index that will have an earlier shard index than the index with the desired result when referenced + # via the alias + - do: + indices.create: + index: first-test-index + body: + mappings: + properties: + text: + type: text + topic: + type: keyword + subtopic: + type: keyword + + - do: + indices.create: + index: second-test-index + body: + settings: + number_of_shards: 2 + number_of_replicas: 0 + mappings: + properties: + text: + type: text + topic: + type: keyword + subtopic: + type: keyword + + - do: + indices.put_alias: + index: first-test-index + name: test-alias + + - do: + indices.put_alias: + index: second-test-index + name: test-alias + + - do: + index: + index: second-test-index + id: doc_1 + body: + text: "As seen from Earth, a solar eclipse happens when the Moon is directly between the Earth and the Sun." + topic: [ "science" ] + subtopic: [ "technology" ] + refresh: true + + - do: + search: + index: test-alias + body: + track_total_hits: true + retriever: + text_similarity_reranker: + retriever: + standard: + query: + term: + topic: "science" + rank_window_size: 10 + inference_id: my-rerank-model + inference_text: "How often does the moon hide the sun?" + field: text + size: 10 + + - match: { hits.total.value: 1 } + - length: { hits.hits: 1 } + - match: { hits.hits.0._id: "doc_1" } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java index f4fa2a29d79a0..cdf02273b9df5 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.monitor.metrics.IndexModeStatsActionType; @@ -29,8 +30,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage; -import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING; - public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction { private final ClusterService clusterService; private final FeatureService featureService; @@ -71,7 +70,7 @@ protected void masterOperation( for (IndexMetadata indexMetadata : state.metadata()) { if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) { numIndices++; - if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) { numIndicesWithSyntheticSources++; } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 3316ac3603e9e..633dd6123c088 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -117,7 +117,7 @@ && matchesLogsPattern(dataStreamName)) { if (settingsBuilder == null) { settingsBuilder = Settings.builder(); } - settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + settingsBuilder.put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); } } @@ -206,7 +206,7 @@ MappingHints getMappingHints( var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); var indexMode = tmpIndexMetadata.getIndexMode(); boolean hasSyntheticSourceUsage = false; - if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) + if (IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) || indexMode == IndexMode.LOGSDB || indexMode == IndexMode.TIME_SERIES) { // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default @@ -214,7 +214,7 @@ MappingHints getMappingHints( // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); + var sourceMode = IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(indexTemplateAndCreateRequestSettings).isEmpty() == false) { // Custom sort config, no point for further checks on [host.name] field. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java index f8f307b572f33..b2a533f6b76c9 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LegacyLicenceIntegrationTests.java @@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.license.AbstractLicensesIntegrationTestCase; import org.elasticsearch.license.GetFeatureUsageRequest; import org.elasticsearch.license.GetFeatureUsageResponse; @@ -104,11 +104,11 @@ public void testSyntheticSourceUsageTracksBothLegacyAndRegularFeature() throws E } private void createIndexWithSyntheticSourceAndAssertExpectedType(String indexName, String expectedType) { - var settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); + var settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "synthetic").build(); createIndex(indexName, settings); var response = admin().indices().getSettings(new GetSettingsRequest().indices(indexName)).actionGet(); assertThat( - response.getIndexToSettings().get(indexName).get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), + response.getIndexToSettings().get(indexName).get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo(expectedType) ); } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 71d52842ecbae..77319a881f1e5 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -640,7 +640,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws ); Metadata metadata = mb.build(); LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); - Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + Settings settings = builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) .build(); Settings result = provider.getAdditionalIndexSettings( @@ -666,7 +666,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(2)); result = provider.getAdditionalIndexSettings( @@ -679,7 +679,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(3)); result = provider.getAdditionalIndexSettings( @@ -692,7 +692,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws List.of() ); assertThat(result.size(), equalTo(3)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); assertThat(newMapperServiceCounter.get(), equalTo(4)); @@ -747,7 +747,7 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch( List.of() ); assertThat(result.size(), equalTo(4)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(SourceFieldMapper.Mode.STORED, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index 4921344278845..7fa2f11880f40 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -10,9 +10,9 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.license.XPackLicenseState; @@ -58,16 +58,16 @@ public void setup() throws Exception { } public void testGetAdditionalIndexSettingsDefault() { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } public void testGetAdditionalIndexSettingsApm() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-apm.app.test"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -75,7 +75,7 @@ public void testGetAdditionalIndexSettingsApm() throws IOException { } public void testGetAdditionalIndexSettingsProfiling() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); for (String dataStreamName : new String[] { "profiling-metrics", "profiling-events" }) { String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, null, null, null, settings, List.of()); @@ -89,7 +89,7 @@ public void testGetAdditionalIndexSettingsProfiling() throws IOException { } public void testGetAdditionalIndexSettingsTsdb() throws IOException { - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); @@ -118,11 +118,11 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception true ); - Settings settings = Settings.builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); + Settings settings = Settings.builder().put(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), "SYNTHETIC").build(); String dataStreamName = "metrics-my-app"; String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); var result = provider.getAdditionalIndexSettings(indexName, dataStreamName, IndexMode.TIME_SERIES, null, null, settings, List.of()); assertThat(result.size(), equalTo(1)); - assertThat(result.get(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); + assertThat(result.get(IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey()), equalTo("STORED")); } } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml index 1fd10c2028fd2..b802c8958f421 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/30_logsdb_default_mapping.yml @@ -2,9 +2,6 @@ create logsdb data stream with host.name as keyword: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: cluster.put_component_template: @@ -38,9 +35,6 @@ create logsdb data stream with host.name as keyword: create logsdb data stream with host.name as keyword and timestamp as date: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: cluster.put_component_template: @@ -76,9 +70,6 @@ create logsdb data stream with host.name as keyword and timestamp as date: create logsdb data stream with host.name as integer and timestamp as date: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: cluster.put_component_template: @@ -114,9 +105,6 @@ create logsdb data stream with host.name as integer and timestamp as date: create logsdb data stream with host as keyword: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: cluster.put_component_template: @@ -150,9 +138,6 @@ create logsdb data stream with host as keyword: create logsdb data stream with host as text and multi fields: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: cluster.put_component_template: @@ -193,9 +178,6 @@ create logsdb data stream with host as text and multi fields: create logsdb data stream with host as text: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: ["mapper.keyword_normalizer_synthetic_source"] - reason: "Support for normalizer on keyword fields" - do: cluster.put_component_template: @@ -232,9 +214,6 @@ create logsdb data stream with host as text: create logsdb data stream with host as text and name as double: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: ["mapper.keyword_normalizer_synthetic_source"] - reason: "Support for normalizer on keyword fields" - do: cluster.put_component_template: @@ -274,9 +253,6 @@ create logsdb data stream with host as text and name as double: create logsdb data stream with timestamp object mapping: - requires: test_runner_features: [ "allowed_warnings" ] - - requires: - cluster_features: ["mapper.keyword_normalizer_synthetic_source"] - reason: "Support for normalizer on keyword fields" - do: cluster.put_component_template: @@ -315,9 +291,6 @@ create logsdb data stream with timestamp object mapping: create logsdb data stream with custom sorting without host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -365,9 +338,6 @@ create logsdb data stream with custom sorting without host.name: create logsdb data stream with custom sorting and host object: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -424,9 +394,6 @@ create logsdb data stream with custom sorting and host object: create logsdb data stream with custom sorting and dynamically mapped host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -489,9 +456,6 @@ create logsdb data stream with custom sorting and dynamically mapped host.name: create logsdb data stream with custom sorting and host.name object: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -545,9 +509,6 @@ create logsdb data stream with custom sorting and host.name object: create logsdb data stream with default sorting on malformed host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -609,9 +570,6 @@ create logsdb data stream with default sorting on malformed host.name: create logsdb data stream with custom sorting and host.name date field: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -662,9 +620,6 @@ create logsdb data stream with custom sorting and host.name date field: create logsdb data stream with custom sorting and missing host.name field mapping: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -713,9 +668,6 @@ create logsdb data stream with custom sorting and missing host.name field mappin create logsdb data stream with custom sorting and host.name field without doc values: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -755,9 +707,6 @@ create logsdb data stream with custom sorting and host.name field without doc va create logsdb data stream with incompatible ignore_above on host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -803,9 +752,6 @@ create logsdb data stream with incompatible ignore_above on host.name: create logsdb data stream with no sorting and host.name as text: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -846,9 +792,6 @@ create logsdb data stream with no sorting and host.name as text: create logsdb data stream without index sorting and ignore_above on host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -892,9 +835,6 @@ create logsdb data stream without index sorting and ignore_above on host.name: create logsdb data stream with host.name as alias and sorting on it: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -930,9 +870,6 @@ create logsdb data stream with host.name as alias and sorting on it: create logsdb data stream with multi-fields on host.name: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -977,9 +914,6 @@ create logsdb data stream with multi-fields on host.name: create logsdb data stream with multi-fields on host.name and no sorting: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -1021,9 +955,6 @@ create logsdb data stream with multi-fields on host.name and no sorting: create logsdb data stream with custom empty sorting: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: @@ -1068,9 +999,6 @@ create logsdb data stream with custom empty sorting: create logsdb data stream with custom sorting on timestamp: - skip: features: [ "allowed_warnings" ] - - requires: - cluster_features: [ "mapper.keyword_normalizer_synthetic_source" ] - reason: support for normalizer on keyword fields - do: allowed_warnings: diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml index 22f69e30650fd..eb7c1df91de60 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml @@ -5,10 +5,6 @@ setup: --- synthetic source: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires synthetic source support - - do: indices.create: index: synthetic_source_test @@ -51,10 +47,6 @@ synthetic source: --- synthetic source with copy_to: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: synthetic_source_test @@ -111,10 +103,6 @@ synthetic source with copy_to: --- synthetic source with disabled doc_values: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires disabled doc_values support in synthetic source - - do: indices.create: index: synthetic_source_test diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml index 3095a19fa29d0..0989a92f2ca01 100644 --- a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml +++ b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml @@ -69,10 +69,6 @@ script values: --- synthetic source with copy_to: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: synthetic_source_test diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java index b460c6abfeee4..f1b14658c6b8e 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java @@ -110,7 +110,8 @@ public void testMappingsCopiedFromSource() { client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex)) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -224,7 +225,7 @@ public void testMappingsOverridden() { ) ); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(destIndex)).actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index 0902f6ce6468a..b310d332fdd5c 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.migrate.action; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; @@ -96,7 +97,7 @@ public void testDestIndexDeletedIfExists() throws Exception { assertHitCount(prepareSearch(destIndex).setSize(0), 0); } - public void testDestIndexNameSet() throws Exception { + public void testDestIndexNameSet_noDotPrefix() throws Exception { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); @@ -110,6 +111,20 @@ public void testDestIndexNameSet() throws Exception { assertEquals(expectedDestIndexName, response.getDestIndex()); } + public void testDestIndexNameSet_withDotPrefix() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get(); + + // call reindex + var response = client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) + .actionGet(); + + var expectedDestIndexName = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndexName, response.getDestIndex()); + } + public void testDestIndexContainsDocs() throws Exception { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); @@ -199,7 +214,8 @@ public void testMappingsAddedToDestIndex() throws Exception { .actionGet() .getDestIndex(); - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); Map mappings = mappingsResponse.mappings(); var destMappings = mappings.get(destIndex).sourceAsMap(); var sourceMappings = mappings.get(sourceIndex).sourceAsMap(); @@ -209,16 +225,46 @@ public void testMappingsAddedToDestIndex() throws Exception { assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); } - public void testReadOnlyAddedBack() { + public void testFailIfMetadataBlockSet() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + try { + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + } catch (ElasticsearchException e) { + assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); + } + + cleanupMetadataBlocks(sourceIndex); + } + + public void testFailIfReadBlockSet() { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); + + try { + client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet(); + } catch (ElasticsearchException e) { + assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); + } + + cleanupMetadataBlocks(sourceIndex); + } + + public void testReadOnlyBlocksNotAddedBack() { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); - // Create source index with read-only and/or block-writes var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - boolean isReadOnly = randomBoolean(); - boolean isBlockWrites = randomBoolean(); var settings = Settings.builder() - .put(IndexMetadata.SETTING_READ_ONLY, isReadOnly) - .put(IndexMetadata.SETTING_BLOCKS_WRITE, isBlockWrites) + .put(IndexMetadata.SETTING_READ_ONLY, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) .build(); indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).actionGet(); @@ -227,13 +273,13 @@ public void testReadOnlyAddedBack() { .actionGet() .getDestIndex(); - // assert read-only settings added back to dest index var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); - assertEquals(isReadOnly, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); - assertEquals(isBlockWrites, Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); + assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); - removeReadOnly(sourceIndex); - removeReadOnly(destIndex); + cleanupMetadataBlocks(sourceIndex); + cleanupMetadataBlocks(destIndex); } public void testUpdateSettingsDefaultsRestored() { @@ -305,7 +351,8 @@ public void testSettingsAndMappingsFromTemplate() throws IOException { // verify mappings from templates copied to dest index { - var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet(); + var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sourceIndex, destIndex)) + .actionGet(); var destMappings = mappingsResponse.mappings().get(destIndex).sourceAsMap(); var sourceMappings = mappingsResponse.mappings().get(sourceIndex).sourceAsMap(); assertEquals(sourceMappings, destMappings); @@ -412,10 +459,11 @@ public void testTsdbStartEndSet() throws Exception { // TODO check other IndexMetadata fields that need to be fixed after the fact // TODO what happens if don't have necessary perms for a given index? - private static void removeReadOnly(String index) { + private static void cleanupMetadataBlocks(String index) { var settings = Settings.builder() - .put(IndexMetadata.SETTING_READ_ONLY, false) - .put(IndexMetadata.SETTING_BLOCKS_WRITE, false) + .putNull(IndexMetadata.SETTING_READ_ONLY) + .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) + .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) .build(); assertAcked(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index)).actionGet()); } @@ -446,4 +494,32 @@ private static String getIndexUUID(String index) { .get(index) .get(IndexMetadata.SETTING_INDEX_UUID); } + + public void testGenerateDestIndexName_noDotPrefix() { + String sourceIndex = "sourceindex"; + String expectedDestIndex = "migrated-sourceindex"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withDotPrefix() { + String sourceIndex = ".sourceindex"; + String expectedDestIndex = ".migrated-sourceindex"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withHyphen() { + String sourceIndex = "source-index"; + String expectedDestIndex = "migrated-source-index"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } + + public void testGenerateDestIndexName_withUnderscore() { + String sourceIndex = "source_index"; + String expectedDestIndex = "migrated-source_index"; + String actualDestIndex = ReindexDataStreamIndexTransportAction.generateDestIndexName(sourceIndex); + assertEquals(expectedDestIndex, actualDestIndex); + } } diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index d86885ce0fbe4..7bb440bc52a15 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -40,7 +40,6 @@ import java.util.Locale; import java.util.Map; -import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY; import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< @@ -93,13 +92,22 @@ protected void doExecute( ); } + if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) { + var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + return; + } + if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) { + var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName); + listener.onFailure(new ElasticsearchException(errorMessage)); + return; + } + SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l, taskId)) .andThen(l -> deleteDestIfExists(destIndexName, l, taskId)) .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) .andThen(l -> reindex(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> copyOldSourceSettingsToDest(settingsBefore, destIndexName, l, taskId)) - .andThen(l -> addBlockIfFromSource(WRITE, settingsBefore, destIndexName, l, taskId)) - .andThen(l -> addBlockIfFromSource(READ_ONLY, settingsBefore, destIndexName, l, taskId)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) .addListener(listener); } @@ -120,7 +128,8 @@ public void onResponse(AddIndexBlockResponse response) { @Override public void onFailure(Exception e) { if (e instanceof ClusterBlockException || e.getCause() instanceof ClusterBlockException) { - // It's fine if block-writes is already set + // Could fail with a cluster block exception if read-only or read-only-allow-delete is already set + // In this case, we can proceed listener.onResponse(null); } else { listener.onFailure(e); @@ -146,10 +155,12 @@ private void createIndex( ) { logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName()); - // override read-only settings if they exist var removeReadOnlyOverride = Settings.builder() + // remove read-only settings if they exist .putNull(IndexMetadata.SETTING_READ_ONLY) + .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) + // settings to optimize reindex .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build(); @@ -192,6 +203,18 @@ private void addBlockIfFromSource( } } + private void updateSettings( + String index, + Settings.Builder settings, + ActionListener listener, + TaskId parentTaskId + ) { + var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), index); + updateSettingsRequest.setParentTask(parentTaskId); + var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", index); + client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + } + private void copyOldSourceSettingsToDest( Settings settingsBefore, String destIndexName, @@ -199,15 +222,10 @@ private void copyOldSourceSettingsToDest( TaskId parentTaskId ) { logger.debug("Updating settings on destination index after reindex completes"); - var settings = Settings.builder(); copySettingOrUnset(settingsBefore, settings, IndexMetadata.SETTING_NUMBER_OF_REPLICAS); copySettingOrUnset(settingsBefore, settings, IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()); - - var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), destIndexName); - updateSettingsRequest.setParentTask(parentTaskId); - var errorMessage = String.format(Locale.ROOT, "Could not update settings on index [%s]", destIndexName); - client.admin().indices().updateSettings(updateSettingsRequest, failIfNotAcknowledged(listener, errorMessage)); + updateSettings(destIndexName, settings, listener, parentTaskId); } private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder builder, String setting) { @@ -221,8 +239,12 @@ private static void copySettingOrUnset(Settings settingsBefore, Settings.Builder } } - public static String generateDestIndexName(String sourceIndex) { - return "migrated-" + sourceIndex; + static String generateDestIndexName(String sourceIndex) { + String prefix = "migrated-"; + if (sourceIndex.startsWith(".")) { + return "." + prefix + sourceIndex.substring(1); + } + return prefix + sourceIndex; } private static ActionListener failIfNotAcknowledged( diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java index 5287d149fae3d..367c1cee8b0ee 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java @@ -11,6 +11,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.ReferenceDocs; @@ -80,6 +82,10 @@ public class DatafeedJobsIT extends MlNativeAutodetectIntegTestCase { public void cleanup() { updateClusterSettings(Settings.builder().putNull("logger.org.elasticsearch.xpack.ml.datafeed")); cleanUp(); + // Race conditions between closing and killing tasks in these tests, + // sometimes result in lingering persistent tasks (such as "_close"), + // which cause subsequent tests to fail. + client().execute(TransportCancelTasksAction.TYPE, new CancelTasksRequest()); } public void testLookbackOnly() throws Exception { diff --git a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java index ff6c3c8444783..7ce0fbe760644 100644 --- a/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java +++ b/x-pack/plugin/ml/src/internalClusterTest/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java @@ -255,7 +255,7 @@ public void testMultipleSimultaneousJobCreations() { // Assert that the mappings contain all the additional fields: field1, field2, field3, etc. String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT; - GetMappingsRequest request = new GetMappingsRequest().indices(sharedResultsIndex); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(sharedResultsIndex); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); @@ -506,7 +506,7 @@ public void testGetDataCountsModelSizeAndTimingStatsWithSomeDocs() throws Except } private Map getIndexMappingProperties(String index) { - GetMappingsRequest request = new GetMappingsRequest().indices(index); + GetMappingsRequest request = new GetMappingsRequest(TEST_REQUEST_TIMEOUT).indices(index); GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet(); Map indexMappings = response.getMappings(); assertNotNull(indexMappings); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java index 759538b4cdc63..dea5e891bd8dd 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDataFrameAnalyticsAction.java @@ -223,7 +223,7 @@ public void onFailure(Exception e) { ); // Get start context - getStartContext(request.getId(), task, startContextListener); + getStartContext(request.getId(), task, startContextListener, request.masterNodeTimeout()); } private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext, ActionListener listener) { @@ -264,7 +264,7 @@ private void estimateMemoryUsageAndUpdateMemoryTracker(StartContext startContext } - private void getStartContext(String id, Task task, ActionListener finalListener) { + private void getStartContext(String id, Task task, ActionListener finalListener, TimeValue masterTimeout) { ParentTaskAssigningClient parentTaskClient = new ParentTaskAssigningClient(client, task.getParentTaskId()); @@ -320,6 +320,7 @@ private void getStartContext(String id, Task task, ActionListener .andThen( (l, startContext) -> MappingsMerger.mergeMappings( parentTaskClient, + masterTimeout, startContext.config.getHeaders(), startContext.config.getSource(), l.map(ignored -> startContext) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java index ed12f54ab86b5..ba69c8733dd13 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DestinationIndex.java @@ -38,6 +38,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsDest; import org.elasticsearch.xpack.core.ml.dataframe.analyses.RequiredField; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.MachineLearning; import java.time.Clock; import java.util.Arrays; @@ -147,6 +148,7 @@ private static void prepareCreateIndexRequest( final Settings settings = settings(settingsResponse, destIndexAllowedSettings); MappingsMerger.mergeMappings( client, + MachineLearning.HARD_CODED_MACHINE_LEARNING_MASTER_NODE_TIMEOUT, config.getHeaders(), config.getSource(), delegate.delegateFailureAndWrap( diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java index 8fee0160cf92e..817020d788554 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/MappingsMerger.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsSource; @@ -34,6 +35,7 @@ private MappingsMerger() {} public static void mergeMappings( Client client, + TimeValue masterTimeout, Map headers, DataFrameAnalyticsSource source, ActionListener listener @@ -43,7 +45,7 @@ public static void mergeMappings( listener::onFailure ); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest(masterTimeout); getMappingsRequest.indices(source.getIndex()); ClientHelper.executeWithHeadersAsync(headers, ML_ORIGIN, client, GetMappingsAction.INSTANCE, getMappingsRequest, mappingsListener); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java index 51b3e0b55d75b..23b12a23e7495 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java @@ -126,6 +126,7 @@ import org.elasticsearch.xpack.core.ml.stats.StatsAccumulator; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.job.categorization.GrokPatternCreator; import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; @@ -392,7 +393,10 @@ private void getLatestIndexMappingsAndAddTerms(String indexName, Collection ssb.parseXContent(parser, true, nf -> nf == RetrieverBuilder.RETRIEVERS_SUPPORTED) - ); + ParsingException iae = expectThrows(ParsingException.class, () -> ssb.parseXContent(parser, true, Predicates.never())); assertEquals("unknown retriever [rrf]", iae.getMessage()); } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 3035bb98a3a93..3d04c9ed0739d 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -199,7 +199,7 @@ static void createIndex( ActionListener.wrap(createIndexResponse -> startPersistentTask(job, listener, persistentTasksService), e -> { if (e instanceof ResourceAlreadyExistsException) { logger.debug("Rolled index already exists for rollup job [" + job.getConfig().getId() + "], updating metadata."); - updateMapping(job, listener, persistentTasksService, client, logger); + updateMapping(job, listener, persistentTasksService, client, logger, request.masterNodeTimeout()); } else { String msg = "Could not create index for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); @@ -249,7 +249,8 @@ static void updateMapping( ActionListener listener, PersistentTasksService persistentTasksService, Client client, - Logger logger + Logger logger, + TimeValue masterTimeout ) { final String indexName = job.getConfig().getRollupIndex(); @@ -304,7 +305,7 @@ static void updateMapping( ); }; - GetMappingsRequest request = new GetMappingsRequest(); + GetMappingsRequest request = new GetMappingsRequest(masterTimeout); client.execute(GetMappingsAction.INSTANCE, request, ActionListener.wrap(getMappingResponseHandler, e -> { String msg = "Could not update mappings for rollup job [" + job.getConfig().getId() + "]"; logger.error(msg); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java index ee8b4c79d1893..fed2439e513c8 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/PutJobStateMachineTests.java @@ -166,7 +166,14 @@ public void testGetMappingFails() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -201,7 +208,14 @@ public void testNoMetadataInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -238,7 +252,14 @@ public void testMetadataButNotRollup() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -272,7 +293,14 @@ public void testJobAlreadyInMapping() { return null; }).when(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), requestCaptor.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); } @@ -318,7 +346,14 @@ public void testAddJobToMapping() { return null; }).when(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), requestCaptor2.capture()); - TransportPutRollupJobAction.updateMapping(job, testListener, mock(PersistentTasksService.class), client, logger); + TransportPutRollupJobAction.updateMapping( + job, + testListener, + mock(PersistentTasksService.class), + client, + logger, + TEST_REQUEST_TIMEOUT + ); verify(client).execute(eq(GetMappingsAction.INSTANCE), any(GetMappingsRequest.class), any()); verify(client).execute(eq(TransportPutMappingAction.TYPE), any(PutMappingRequest.class), any()); } diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java index 47c7ac8241fcd..19d742e32aa08 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1EnrichUnavailableRemotesIT.java @@ -24,11 +24,13 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class CrossClusterEsqlRCS1EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { private static final AtomicBoolean SSL_ENABLED_REF = new AtomicBoolean(); @@ -185,8 +187,7 @@ private void esqlEnrichWithSkipUnavailableTrue() throws Exception { Map failuresMap = (Map) remoteClusterFailures.get(0); Map reason = (Map) failuresMap.get("reason"); - assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); - assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception")); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); @@ -202,7 +203,11 @@ private void esqlEnrichWithSkipUnavailableFalse() throws Exception { String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; ResponseException ex = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(query))); - assertThat(ex.getMessage(), containsString("connect_transport_exception")); + + assertThat( + ex.getMessage(), + anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception")) + ); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java index da59e3c772736..3f068cf3a04cc 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS2EnrichUnavailableRemotesIT.java @@ -26,11 +26,13 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class CrossClusterEsqlRCS2EnrichUnavailableRemotesIT extends AbstractRemoteClusterSecurityTestCase { private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); @@ -205,8 +207,7 @@ private void esqlEnrichWithSkipUnavailableTrue() throws Exception { Map failuresMap = (Map) remoteClusterFailures.get(0); Map reason = (Map) failuresMap.get("reason"); - assertThat(reason.get("type").toString(), equalTo("connect_transport_exception")); - assertThat(reason.get("reason").toString(), containsString("Unable to connect to [my_remote_cluster]")); + assertThat(reason.get("type").toString(), oneOf("node_disconnected_exception", "connect_transport_exception")); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); @@ -222,7 +223,10 @@ private void esqlEnrichWithSkipUnavailableFalse() throws Exception { String query = "FROM to-be-enr*,my_remote_cluster:to-be-enr* | ENRICH " + randomFrom(modes) + ":employees-policy | LIMIT 10"; ResponseException ex = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(esqlRequest(query))); - assertThat(ex.getMessage(), containsString("connect_transport_exception")); + assertThat( + ex.getMessage(), + anyOf(containsString("connect_transport_exception"), containsString("node_disconnected_exception")) + ); } finally { fulfillingCluster.start(); closeFulfillingClusterClient(); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 57d18abaf1a92..f7361480f0a51 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -287,28 +287,28 @@ public void testGetMappingsIsFiltered() { { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field2"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1"); } { GetMappingsResponse getMappingsResponse = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("test").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "test").get(); assertExpectedMetadataFields(getMappingsResponse.getMappings(), "field1", "field2"); } } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index bc730b5695c19..b501ba69d673b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -166,7 +166,7 @@ public void testGetMappings() throws Exception { GetMappingsResponse response = client().filterWithHeader( singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD)) - ).admin().indices().prepareGetMappings("logstash-*").get(); + ).admin().indices().prepareGetMappings(TEST_REQUEST_TIMEOUT, "logstash-*").get(); Map mappingsMap = response.getMappings(); assertNotNull(mappingsMap); assertNotNull(mappingsMap.get(index)); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml index 2576a51e8b80e..3723f6cd4d450 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/aggregate-metrics/100_synthetic_source.yml @@ -61,10 +61,6 @@ aggregate_metric_double: --- aggregate_metric_double with ignore_malformed: - - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: requires tracking ignored source - - do: indices.create: index: test diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml index 5e9faa84ee088..10556a49bcb4e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml @@ -5,10 +5,6 @@ setup: --- "geo_shape": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test @@ -72,10 +68,6 @@ setup: --- "geo_shape with ignore_malformed": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test @@ -156,10 +148,6 @@ setup: --- "shape": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test @@ -223,10 +211,6 @@ setup: --- "shape with ignore_malformed": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test @@ -424,8 +408,6 @@ setup: --- "geo_point with ignore_malformed": - requires: - cluster_features: ["mapper.track_ignored_source"] - reason: introduced in 8.15.0 test_runner_features: close_to - do: @@ -504,10 +486,6 @@ setup: --- "point": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test @@ -601,10 +579,6 @@ setup: --- "point with ignore_malformed": - - requires: - cluster_features: ["mapper.source.synthetic_source_fallback"] - reason: introduced in 8.15.0 - - do: indices.create: index: test diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml index 252bafbdbe15a..12c2a9858a16b 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/20_ignore_above_stored_source.yml @@ -1,8 +1,5 @@ --- wildcard field type ignore_above: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml index 26beb3aa19075..5076e7556d911 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/wildcard/30_ignore_above_synthetic_source.yml @@ -5,9 +5,6 @@ setup: --- wildcard field type ignore_above: - - requires: - cluster_features: [ "mapper.ignore_above_index_level_setting" ] - reason: introduce ignore_above index level setting - do: indices.create: index: test diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java index ffac36846414e..ea9d76fb643fb 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateHttpMappingsTests.java @@ -172,7 +172,8 @@ public void testExceptionMapping() throws Exception { // ensure that enabled is set to false List indexed = new ArrayList<>(); - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(HistoryStoreField.INDEX_PREFIX + "*").get(); + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, HistoryStoreField.INDEX_PREFIX + "*") + .get(); for (MappingMetadata mapping : mappingsResponse.getMappings().values()) { Map docMapping = mapping.getSourceAsMap(); if (abortAtInput) { diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java index c78315383df2f..34d38dfde064d 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTimeMappingsTests.java @@ -46,7 +46,7 @@ public void testTimeFields() throws Exception { assertWatchWithMinimumActionsCount("_id", ExecutionState.EXECUTED, 1); assertBusy(() -> { - GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings() + GetMappingsResponse mappingsResponse = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT) .setIndicesOptions(IndicesOptions.strictExpandHidden()) .get(); assertThat(mappingsResponse, notNullValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java index 1bcdd060994ce..732b0800b4316 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/integration/HistoryIntegrationTests.java @@ -120,7 +120,7 @@ public void testFailedInputResultWithDotsInFieldNameGetsStored() throws Exceptio assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -162,7 +162,7 @@ public void testPayloadInputWithDotsInFieldNameWorks() throws Exception { assertBusy(() -> { assertHitCount(getWatchHistory(), 1); }); // as fields with dots are allowed in 5.0 again, the mapping must be checked in addition - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource source = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON @@ -225,7 +225,7 @@ public void testThatHistoryContainsStatus() throws Exception { assertBusy(() -> { // also ensure that the status field is disabled in the watch history - GetMappingsResponse response = indicesAdmin().prepareGetMappings(".watcher-history*").get(); + GetMappingsResponse response = indicesAdmin().prepareGetMappings(TEST_REQUEST_TIMEOUT, ".watcher-history*").get(); XContentSource mappingSource = new XContentSource( response.getMappings().values().iterator().next().source().uncompressed(), XContentType.JSON diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml index 14e4a6f5aaef8..4eaf89d0bf67a 100644 --- a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml @@ -5,10 +5,6 @@ setup: --- synthetic source: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires synthetic source support - - do: indices.create: index: synthetic_source_test @@ -47,10 +43,6 @@ synthetic source: --- synthetic source with copy_to: - - requires: - cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] - reason: requires copy_to support in synthetic source - - do: indices.create: index: synthetic_source_test diff --git a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml index 1b435c551fbe9..41bc0a96e737e 100644 --- a/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml +++ b/x-pack/qa/multi-cluster-search-security/legacy-with-basic-license/src/test/resources/rest-api-spec/test/querying_cluster/80_esql.yml @@ -92,8 +92,6 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: @@ -104,12 +102,10 @@ teardown: - do: catch: bad_request - allowed_warnings: - - "Line 1:21: Square brackets '[]' need to be removed in FROM METADATA declaration" headers: { Authorization: "Basic am9lOnMza3JpdC1wYXNzd29yZA==" } esql.query: body: - query: 'FROM *:esql*,esql_* [METADATA _index] | sort cost | KEEP _index, tag, cost | LIMIT 10' + query: 'FROM *:esql*,esql_* METADATA _index | sort cost | KEEP _index, tag, cost | LIMIT 10' filter: range: since: